diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv index 08371f232247c5d17479baeec6781f0cd122cc83..6b993632bfa140bc1f451862d78d6cc20fa3f913 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv @@ -78,3 +78,147 @@ 5, 44671 6, 44671 7, 44671 +0, 47221 +1, 46653 +2, 46653 +3, 47353 +4, 47353 +5, 47353 +6, 47353 +7, 47353 +0, 47501 +1, 46933 +2, 46933 +3, 47633 +4, 47633 +5, 47633 +6, 47633 +7, 47633 +0, 47783 +1, 47207 +2, 47221 +3, 47919 +4, 47903 +5, 47955 +6, 47909 +7, 47909 +0, 48535 +1, 47967 +2, 47967 +3, 48667 +4, 48667 +5, 48667 +6, 48667 +7, 48667 +0, 48983 +1, 48415 +2, 48415 +3, 49115 +4, 49115 +5, 49115 +6, 49115 +7, 49115 +0, 51025 +1, 50457 +2, 49521 +3, 51157 +4, 50705 +5, 51157 +6, 51157 +7, 51157 +0, 51025 +1, 50457 +2, 50457 +3, 51157 +4, 51157 +5, 51157 +6, 51157 +7, 51157 +0, 82459 +1, 82465 +2, 82465 +3, 82463 +4, 82463 +5, 82463 +6, 82463 +7, 82463 +0, 82847 +1, 82853 +2, 82853 +3, 82851 +4, 82851 +5, 82851 +6, 82851 +7, 82851 +0, 84033 +1, 84039 +2, 84039 +3, 84037 +4, 84037 +5, 84037 +6, 84037 +7, 84037 +0, 84033 +1, 84039 +2, 84039 +3, 84037 +4, 84037 +5, 84037 +6, 84037 +7, 84037 +0, 84033 +1, 84039 +2, 84039 +3, 84037 +4, 84037 +5, 84037 +6, 84037 +7, 84037 +0, 84527 +1, 84507 +2, 84573 +3, 84515 +4, 84531 +5, 84509 +6, 84531 +7, 85139 +0, 88033 +1, 87199 +2, 88039 +3, 87197 +4, 87197 +5, 87197 +6, 87197 +7, 88037 +0, 88671 +1, 87825 +2, 88681 +3, 87817 +4, 87811 +5, 87819 +6, 87833 +7, 88695 +0, 109033 +1, 108271 +2, 109131 +3, 108257 +4, 108257 +5, 108257 +6, 108257 +7, 109097 +0, 109033 +1, 108271 +2, 109131 +3, 108257 +4, 108257 +5, 108257 +6, 108257 +7, 109097 +0, 2382 +1, 2382 +2, 2638 +3, 107363 +4, 3662 +5, 2382 +6, 2382 +7, 2382 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log index 258552490f565d16eb0eec37b39647944932c1a3..33098ac4ff12876d99ab63a05a34ea7802f30371 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log @@ -1030,3 +1030,4972 @@ WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] glob warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[500, 531, 498, 494, 478, 493, 496, 503, 537, 523, 503, 524, 522, 529, 547, 549, 470, 523, 496, 516, 527, 495, 537, 465, 499, 517, 515, 523, 532, 511, 487, 525, 562, 479, 525, 507, 484, 491, 511, 484, 541, 528, 515, 503, 475, 491, 504, 490, 561, 510, 549, 525, 497, 498, 510, 515, 506, 543, 533, 509, 530, 518, 502, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[526, 517, 542, 515, 524, 467, 514, 504, 511, 490, 510, 486, 535, 506, 498, 508, 519, 528, 526, 502, 490, 532, 533, 513, 526, 525, 533, 512, 530, 558, 499, 504, 520, 526, 483, 468, 501, 497, 516, 501, 510, 517, 490, 558, 493, 520, 531, 523, 511, 520, 469, 494, 482, 506, 518, 518, 514, 510, 500, 531, 531, 527, 467, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[522, 470, 513, 496, 511, 439, 532, 541, 533, 494, 523, 515, 477, 507, 508, 507, 496, 490, 522, 499, 509, 486, 539, 573, 494, 512, 506, 510, 542, 514, 522, 533, 544, 499, 549, 538, 525, 511, 521, 530, 548, 502, 502, 486, 514, 497, 496, 554, 501, 496, 510, 499, 543, 526, 485, 513, 539, 481, 538, 540, 526, 475, 474, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[495, 491, 538, 524, 525, 498, 551, 505, 528, 486, 516, 462, 516, 519, 545, 555, 527, 515, 517, 506, 516, 542, 545, 517, 492, 503, 529, 531, 522, 491, 528, 521, 519, 494, 512, 487, 446, 494, 497, 470, 533, 494, 474, 565, 505, 528, 507, 501, 508, 492, 505, 518, 538, 537, 543, 520, 467, 525, 500, 497, 496, 513, 508, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[521, 543, 508, 487, 504, 503, 504, 531, 500, 514, 541, 538, 553, 517, 530, 467, 512, 515, 517, 513, 527, 487, 486, 489, 483, 527, 523, 524, 498, 508, 507, 489, 497, 524, 540, 515, 488, 525, 495, 524, 500, 504, 525, 529, 473, 502, 522, 516, 541, 478, 494, 498, 472, 544, 495, 561, 513, 521, 557, 512, 523, 514, 507, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[508, 521, 525, 467, 498, 544, 529, 488, 492, 527, 529, 508, 494, 501, 507, 506, 544, 523, 543, 527, 536, 493, 517, 505, 537, 493, 488, 507, 505, 527, 522, 535, 516, 473, 518, 501, 516, 486, 537, 510, 517, 481, 498, 526, 509, 494, 484, 524, 545, 504, 493, 508, 525, 524, 481, 502, 511, 527, 529, 544, 527, 497, 455, 560] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[486, 531, 540, 506, 545, 501, 561, 527, 519, 480, 488, 505, 534, 488, 534, 492, 539, 525, 484, 523, 498, 503, 493, 502, 504, 516, 551, 481, 515, 528, 498, 514, 489, 464, 546, 537, 540, 494, 505, 525, 517, 466, 547, 508, 529, 524, 562, 465, 499, 470, 473, 552, 536, 526, 529, 474, 553, 507, 472, 504, 507, 495, 529, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[496, 526, 540, 539, 515, 504, 476, 535, 509, 542, 530, 500, 507, 509, 481, 530, 492, 531, 489, 537, 507, 497, 519, 517, 540, 530, 559, 486, 529, 505, 501, 533, 489, 528, 529, 489, 489, 504, 498, 503, 505, 534, 514, 531, 513, 488, 507, 537, 483, 475, 508, 529, 475, 475, 551, 528, 489, 541, 524, 518, 479, 506, 511, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[486, 531, 540, 506, 545, 501, 561, 527, 0, 0, 0, 0, 0, 0, 0, 0, 519, 480, 488, 505, 534, 488, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 539, 525, 484, 523, 498, 503, 483, 502, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 520, 481, 515, 528, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 489, 464, 546, 537, 540, 494, 505, 525, 31, 4, 0, 0, 0, 0, 0, 0, 517, 466, 547, 508, 529, 524, 562, 465, 2, 0, 0, 0, 0, 0, 0, 0, 499, 470, 473, 552, 536, 526, 529, 474, 6, 6, 0, 0, 0, 0, 0, 0, 553, 507, 472, 504, 507, 495, 529, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 4032, 4057, 4076, 4135, 4120, 4071, 4080] avg=4096.0000 max=4197 max_rank=0 min=4032 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 531, 498, 494, 478, 493, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 537, 523, 503, 524, 522, 529, 539, 549, 0, 0, 0, 0, 0, 0, 0, 0, 470, 523, 496, 516, 527, 495, 526, 465, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 486, 523, 532, 511, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 562, 479, 525, 507, 484, 491, 511, 484, 29, 5, 0, 0, 0, 0, 0, 0, 541, 528, 515, 503, 475, 491, 504, 490, 2, 0, 0, 0, 0, 0, 0, 0, 561, 510, 549, 525, 497, 498, 510, 515, 6, 6, 0, 0, 0, 0, 0, 0, 506, 543, 533, 509, 530, 518, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4226, 4018, 4080, 4077, 4049, 4177, 4148] avg=4096.0000 max=4226 max_rank=1 min=3993 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 491, 538, 524, 525, 498, 551, 505, 0, 0, 0, 0, 0, 0, 0, 0, 528, 486, 516, 462, 516, 519, 537, 555, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 517, 506, 516, 542, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 499, 531, 522, 491, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 512, 487, 446, 494, 497, 470, 30, 5, 0, 0, 0, 0, 0, 0, 533, 494, 474, 565, 505, 528, 507, 501, 2, 0, 0, 0, 0, 0, 0, 0, 508, 492, 505, 518, 538, 537, 543, 520, 6, 6, 0, 0, 0, 0, 0, 0, 467, 525, 500, 497, 496, 513, 508, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4119, 4174, 4087, 3954, 4109, 4173, 4025] avg=4096.0000 max=4174 max_rank=2 min=3954 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 517, 542, 515, 524, 467, 514, 504, 6, 0, 0, 0, 0, 0, 0, 0, 511, 490, 510, 486, 535, 506, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 519, 528, 526, 502, 490, 532, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 499, 512, 530, 558, 499, 504, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 483, 468, 501, 497, 516, 501, 34, 9, 0, 0, 0, 0, 0, 0, 510, 517, 490, 558, 493, 520, 531, 523, 7, 1, 0, 0, 0, 0, 0, 0, 511, 520, 469, 494, 482, 506, 518, 518, 10, 10, 0, 0, 0, 0, 0, 0, 514, 503, 500, 531, 531, 527, 467, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4027, 4124, 4153, 4055, 4150, 4038, 4106] avg=4096.0000 max=4153 max_rank=3 min=4027 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[522, 470, 513, 496, 511, 439, 532, 541, 0, 0, 0, 0, 0, 0, 0, 0, 533, 494, 523, 515, 477, 507, 500, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 490, 522, 499, 509, 486, 528, 573, 0, 0, 0, 0, 0, 0, 0, 0, 494, 512, 477, 510, 542, 514, 522, 533, 0, 0, 0, 0, 0, 0, 0, 0, 544, 499, 549, 538, 525, 511, 521, 530, 29, 5, 0, 0, 0, 0, 0, 0, 548, 502, 502, 486, 514, 497, 496, 554, 2, 0, 0, 0, 0, 0, 0, 0, 501, 496, 510, 499, 543, 526, 485, 513, 6, 6, 0, 0, 0, 0, 0, 0, 539, 481, 538, 540, 526, 475, 474, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4056, 4103, 4104, 4251, 4101, 4085, 4044] avg=4096.0000 max=4251 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 543, 508, 487, 504, 503, 504, 531, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 541, 538, 553, 517, 522, 467, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 517, 513, 527, 487, 477, 489, 0, 0, 0, 0, 0, 0, 0, 0, 483, 527, 494, 524, 498, 508, 507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 497, 524, 540, 515, 488, 525, 495, 524, 29, 4, 0, 0, 0, 0, 0, 0, 500, 504, 525, 529, 473, 502, 522, 516, 2, 0, 0, 0, 0, 0, 0, 0, 541, 478, 494, 498, 472, 544, 495, 561, 6, 5, 0, 0, 0, 0, 0, 0, 513, 521, 557, 512, 523, 514, 507, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4152, 4037, 4030, 4141, 4073, 4094, 4140] avg=4096.0000 max=4152 max_rank=1 min=4030 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 526, 540, 539, 515, 504, 476, 535, 0, 0, 0, 0, 0, 0, 0, 0, 509, 542, 530, 500, 507, 509, 473, 530, 0, 0, 0, 0, 0, 0, 0, 0, 492, 531, 489, 537, 507, 497, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 540, 530, 527, 486, 529, 505, 501, 533, 0, 0, 0, 0, 0, 0, 0, 0, 489, 528, 529, 489, 489, 504, 498, 503, 32, 5, 0, 0, 0, 0, 0, 0, 505, 534, 514, 531, 513, 488, 507, 537, 2, 0, 0, 0, 0, 0, 0, 0, 483, 475, 508, 529, 475, 475, 551, 528, 6, 6, 0, 0, 0, 0, 0, 0, 489, 541, 524, 518, 479, 506, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4100, 4078, 4151, 4066, 4131, 4036, 4075] avg=4096.0000 max=4151 max_rank=3 min=4036 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 521, 525, 467, 498, 544, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 492, 527, 529, 508, 494, 501, 499, 506, 0, 0, 0, 0, 0, 0, 0, 0, 544, 523, 543, 527, 536, 493, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 537, 493, 461, 507, 505, 527, 522, 535, 0, 0, 0, 0, 0, 0, 0, 0, 516, 473, 518, 501, 516, 486, 537, 510, 27, 5, 0, 0, 0, 0, 0, 0, 517, 481, 498, 526, 509, 494, 484, 524, 2, 0, 0, 0, 0, 0, 0, 0, 545, 504, 493, 508, 525, 524, 481, 502, 6, 6, 0, 0, 0, 0, 0, 0, 511, 527, 529, 544, 527, 497, 455, 560, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4056, 4177, 4087, 4089, 4035, 4094, 4150] avg=4096.0000 max=4177 max_rank=2 min=4035 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[534, 539, 512, 491, 529, 516, 533, 534, 545, 503, 501, 528, 509, 475, 492, 533, 467, 527, 545, 514, 509, 476, 519, 529, 514, 524, 513, 499, 526, 512, 533, 519, 536, 452, 544, 491, 482, 510, 509, 479, 505, 502, 487, 528, 528, 499, 503, 523, 504, 507, 477, 532, 516, 531, 539, 493, 509, 539, 506, 484, 533, 515, 503, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[534, 507, 540, 496, 516, 479, 532, 504, 516, 463, 503, 514, 511, 497, 536, 488, 533, 526, 510, 495, 478, 483, 506, 541, 510, 500, 498, 481, 523, 545, 519, 534, 488, 508, 538, 524, 513, 511, 521, 511, 521, 512, 523, 537, 506, 507, 543, 497, 479, 518, 499, 518, 492, 532, 549, 492, 507, 540, 536, 536, 553, 479, 501, 459] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[525, 487, 508, 555, 528, 489, 521, 535, 511, 521, 463, 484, 522, 496, 596, 526, 497, 522, 529, 520, 542, 533, 506, 482, 500, 517, 479, 465, 516, 479, 498, 501, 521, 496, 490, 488, 505, 508, 519, 544, 511, 529, 524, 528, 551, 530, 489, 512, 522, 517, 492, 538, 524, 513, 502, 504, 497, 478, 485, 505, 578, 530, 486, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[508, 489, 541, 533, 526, 522, 520, 519, 547, 538, 498, 519, 499, 498, 465, 520, 533, 493, 479, 520, 525, 525, 499, 512, 495, 500, 532, 535, 547, 507, 527, 546, 499, 504, 485, 534, 488, 486, 483, 504, 519, 539, 551, 501, 509, 476, 548, 504, 508, 507, 479, 533, 504, 507, 546, 522, 511, 534, 491, 503, 503, 488, 475, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[536, 517, 547, 542, 529, 488, 487, 561, 498, 497, 477, 495, 509, 499, 495, 480, 472, 553, 530, 520, 520, 504, 530, 491, 473, 501, 510, 519, 505, 512, 496, 528, 483, 562, 476, 533, 549, 523, 466, 513, 495, 548, 520, 497, 517, 529, 545, 516, 471, 511, 505, 533, 498, 488, 498, 531, 511, 533, 497, 528, 503, 522, 529, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[545, 521, 513, 505, 503, 481, 496, 525, 512, 509, 535, 482, 529, 509, 498, 514, 498, 514, 539, 494, 513, 536, 524, 538, 534, 505, 556, 535, 501, 517, 512, 504, 543, 483, 509, 487, 503, 514, 513, 501, 508, 515, 466, 494, 520, 527, 500, 499, 529, 519, 509, 545, 488, 504, 503, 479, 547, 522, 479, 501, 511, 539, 501, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[517, 529, 511, 505, 514, 509, 543, 490, 552, 536, 510, 488, 503, 476, 496, 527, 519, 514, 495, 520, 495, 497, 527, 495, 495, 533, 529, 500, 490, 494, 518, 514, 502, 506, 531, 510, 531, 512, 509, 496, 512, 499, 522, 501, 513, 544, 489, 527, 516, 504, 525, 479, 506, 543, 501, 534, 526, 510, 555, 489, 501, 515, 503, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[533, 509, 506, 505, 517, 497, 508, 507, 555, 508, 518, 521, 494, 528, 518, 511, 488, 509, 499, 483, 531, 515, 499, 490, 498, 482, 531, 480, 506, 527, 511, 514, 512, 512, 517, 492, 532, 500, 506, 534, 536, 523, 489, 508, 527, 514, 512, 509, 515, 510, 478, 532, 503, 502, 550, 513, 510, 486, 553, 503, 516, 497, 536, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 539, 512, 491, 529, 516, 533, 534, 0, 0, 0, 0, 0, 0, 0, 0, 545, 503, 501, 528, 509, 475, 492, 533, 11, 10, 0, 0, 0, 0, 0, 0, 467, 527, 545, 514, 509, 476, 519, 529, 2, 2, 0, 0, 0, 0, 0, 0, 514, 524, 513, 499, 526, 512, 533, 519, 5, 0, 0, 0, 0, 0, 0, 0, 536, 452, 544, 491, 482, 510, 509, 479, 29, 0, 0, 0, 0, 0, 0, 0, 505, 502, 487, 528, 505, 499, 503, 523, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 477, 532, 516, 531, 539, 493, 6, 0, 0, 0, 0, 0, 0, 0, 509, 539, 506, 484, 531, 515, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4107, 4090, 4145, 4032, 4052, 4105, 4089] avg=4096.0000 max=4148 max_rank=0 min=4032 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 507, 540, 496, 516, 479, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 516, 463, 503, 514, 511, 497, 536, 488, 11, 9, 0, 0, 0, 0, 0, 0, 533, 526, 510, 495, 478, 483, 506, 541, 2, 2, 0, 0, 0, 0, 0, 0, 510, 500, 498, 481, 523, 545, 519, 534, 5, 0, 0, 0, 0, 0, 0, 0, 488, 508, 538, 524, 513, 511, 521, 511, 29, 0, 0, 0, 0, 0, 0, 0, 521, 512, 523, 537, 484, 507, 543, 497, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 499, 518, 492, 532, 549, 492, 6, 0, 0, 0, 0, 0, 0, 0, 507, 540, 536, 536, 551, 479, 501, 459, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4048, 4076, 4115, 4143, 4124, 4085, 4109] avg=4096.0000 max=4143 max_rank=4 min=4048 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 517, 547, 542, 529, 488, 487, 561, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 477, 495, 509, 499, 495, 480, 11, 9, 0, 0, 0, 0, 0, 0, 472, 553, 530, 520, 520, 504, 530, 491, 2, 2, 0, 0, 0, 0, 0, 0, 473, 501, 510, 519, 505, 512, 496, 528, 5, 0, 0, 0, 0, 0, 0, 0, 483, 562, 476, 533, 549, 523, 466, 513, 30, 0, 0, 0, 0, 0, 0, 0, 495, 548, 520, 497, 495, 529, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 471, 511, 505, 533, 498, 488, 498, 531, 6, 0, 0, 0, 0, 0, 0, 0, 511, 533, 497, 528, 501, 522, 529, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 3970, 4124, 4049, 4135, 4145, 4041, 4138] avg=4096.0000 max=4166 max_rank=0 min=3970 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 521, 513, 505, 503, 481, 496, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 535, 482, 529, 509, 498, 514, 16, 14, 0, 0, 0, 0, 0, 0, 498, 514, 539, 494, 513, 536, 524, 538, 8, 5, 0, 0, 0, 0, 0, 0, 534, 505, 556, 535, 501, 517, 512, 504, 9, 0, 0, 0, 0, 0, 0, 0, 543, 483, 509, 487, 503, 514, 513, 501, 35, 0, 0, 0, 0, 0, 0, 0, 508, 515, 466, 494, 481, 527, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 509, 545, 488, 504, 503, 479, 8, 0, 0, 0, 0, 0, 0, 0, 547, 522, 479, 501, 506, 539, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4118, 4169, 4173, 4088, 3990, 4084, 4108] avg=4096.0000 max=4173 max_rank=3 min=3990 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[485, 487, 508, 555, 528, 489, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 463, 484, 522, 496, 596, 526, 11, 10, 0, 0, 0, 0, 0, 0, 497, 522, 529, 520, 542, 533, 506, 482, 2, 2, 0, 0, 0, 0, 0, 0, 500, 517, 479, 465, 516, 479, 498, 501, 5, 0, 0, 0, 0, 0, 0, 0, 521, 496, 490, 488, 505, 508, 519, 544, 29, 0, 0, 0, 0, 0, 0, 0, 511, 529, 524, 528, 528, 530, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 492, 538, 524, 513, 502, 504, 6, 0, 0, 0, 0, 0, 0, 0, 497, 478, 485, 505, 576, 530, 486, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4140, 4135, 3960, 4100, 4151, 4118, 4056] avg=4096.0000 max=4151 max_rank=5 min=3960 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[469, 489, 541, 533, 526, 522, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 547, 538, 498, 519, 499, 498, 465, 520, 11, 9, 0, 0, 0, 0, 0, 0, 533, 493, 479, 520, 525, 525, 499, 512, 2, 2, 0, 0, 0, 0, 0, 0, 495, 500, 532, 535, 547, 507, 527, 546, 5, 0, 0, 0, 0, 0, 0, 0, 499, 504, 485, 534, 488, 486, 483, 504, 28, 0, 0, 0, 0, 0, 0, 0, 519, 539, 551, 501, 487, 476, 548, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 479, 533, 504, 507, 546, 522, 6, 0, 0, 0, 0, 0, 0, 0, 511, 534, 491, 503, 501, 488, 475, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4104, 4090, 4194, 4011, 4125, 4112, 4013] avg=4096.0000 max=4194 max_rank=3 min=4011 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[478, 529, 511, 505, 514, 509, 543, 490, 0, 0, 0, 0, 0, 0, 0, 0, 552, 536, 510, 488, 503, 476, 496, 527, 11, 9, 0, 0, 0, 0, 0, 0, 519, 514, 495, 520, 495, 497, 527, 495, 2, 2, 0, 0, 0, 0, 0, 0, 495, 533, 529, 500, 490, 494, 518, 514, 5, 0, 0, 0, 0, 0, 0, 0, 502, 506, 531, 510, 531, 512, 509, 496, 28, 0, 0, 0, 0, 0, 0, 0, 512, 499, 522, 501, 491, 544, 489, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 504, 525, 479, 506, 543, 501, 534, 6, 0, 0, 0, 0, 0, 0, 0, 526, 510, 555, 489, 499, 515, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4108, 4066, 4078, 4125, 4085, 4114, 4113] avg=4096.0000 max=4125 max_rank=4 min=4066 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 506, 505, 517, 497, 508, 507, 0, 0, 0, 0, 0, 0, 0, 0, 555, 508, 518, 521, 494, 528, 518, 511, 11, 9, 0, 0, 0, 0, 0, 0, 488, 509, 499, 483, 531, 515, 499, 490, 2, 2, 0, 0, 0, 0, 0, 0, 498, 482, 531, 480, 506, 527, 511, 514, 5, 0, 0, 0, 0, 0, 0, 0, 512, 512, 517, 492, 532, 500, 506, 534, 29, 0, 0, 0, 0, 0, 0, 0, 536, 523, 489, 508, 505, 514, 512, 509, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 478, 532, 503, 502, 550, 513, 6, 0, 0, 0, 0, 0, 0, 0, 510, 486, 553, 503, 514, 497, 536, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4173, 4018, 4054, 4134, 4096, 4109, 4142] avg=4096.0000 max=4173 max_rank=1 min=4018 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[473, 548, 494, 496, 542, 514, 527, 498, 502, 547, 522, 541, 492, 541, 520, 498, 519, 531, 542, 494, 565, 500, 542, 468, 523, 509, 510, 533, 520, 507, 481, 526, 504, 542, 522, 499, 468, 490, 507, 562, 550, 461, 468, 510, 482, 515, 549, 500, 474, 497, 471, 541, 515, 575, 523, 466, 491, 527, 514, 493, 499, 519, 497, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[545, 517, 550, 503, 475, 521, 558, 473, 505, 486, 518, 527, 519, 524, 503, 502, 494, 513, 510, 491, 483, 523, 498, 497, 515, 529, 533, 484, 498, 501, 528, 515, 503, 491, 494, 518, 505, 528, 517, 512, 484, 535, 526, 548, 514, 486, 507, 515, 509, 556, 510, 491, 512, 537, 491, 500, 523, 543, 568, 484, 506, 511, 510, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[526, 527, 530, 467, 497, 525, 523, 532, 504, 519, 519, 477, 504, 535, 485, 517, 500, 505, 531, 494, 533, 491, 498, 503, 520, 556, 511, 519, 520, 520, 571, 514, 462, 497, 538, 523, 525, 505, 510, 498, 453, 529, 522, 472, 524, 494, 530, 507, 512, 526, 531, 475, 560, 514, 501, 500, 519, 543, 517, 534, 488, 524, 491, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[503, 522, 507, 564, 504, 493, 523, 474, 504, 489, 478, 551, 502, 514, 501, 492, 527, 519, 493, 517, 502, 487, 505, 513, 505, 519, 513, 506, 520, 518, 518, 547, 502, 539, 503, 519, 502, 548, 505, 520, 495, 484, 517, 521, 519, 496, 529, 515, 521, 499, 526, 470, 501, 489, 538, 540, 521, 533, 515, 502, 513, 545, 517, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[529, 574, 495, 510, 531, 526, 523, 510, 551, 524, 508, 504, 527, 507, 504, 536, 515, 521, 513, 521, 490, 488, 503, 505, 523, 522, 503, 504, 538, 495, 485, 499, 483, 512, 488, 503, 512, 542, 529, 521, 546, 489, 521, 502, 491, 508, 523, 514, 503, 530, 522, 520, 486, 502, 477, 512, 533, 493, 514, 508, 518, 521, 447, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[511, 477, 492, 484, 517, 534, 531, 506, 521, 504, 515, 517, 519, 494, 471, 510, 513, 524, 513, 500, 520, 554, 525, 485, 503, 534, 502, 545, 475, 529, 499, 544, 501, 502, 518, 484, 493, 493, 525, 507, 514, 515, 544, 490, 497, 494, 541, 491, 538, 520, 544, 472, 491, 555, 499, 510, 549, 522, 509, 508, 540, 510, 509, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[528, 506, 495, 526, 498, 521, 519, 492, 512, 534, 504, 508, 530, 503, 502, 528, 515, 508, 487, 553, 537, 520, 505, 497, 494, 534, 520, 516, 508, 529, 518, 595, 555, 508, 518, 490, 521, 482, 526, 501, 515, 513, 531, 529, 501, 521, 503, 484, 484, 501, 514, 518, 506, 502, 535, 498, 518, 495, 458, 508, 492, 484, 497, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[517, 502, 481, 490, 493, 498, 483, 538, 538, 504, 486, 536, 535, 542, 499, 496, 496, 531, 543, 517, 479, 522, 519, 502, 517, 530, 494, 542, 511, 521, 508, 572, 500, 511, 468, 554, 485, 496, 532, 480, 521, 539, 516, 511, 560, 524, 498, 528, 514, 481, 530, 533, 523, 475, 529, 496, 478, 482, 546, 461, 515, 492, 538, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[473, 548, 494, 496, 542, 514, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 502, 547, 522, 537, 492, 541, 520, 498, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 542, 494, 565, 500, 542, 468, 4, 2, 0, 0, 0, 0, 0, 0, 523, 509, 510, 533, 520, 507, 481, 476, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 522, 499, 468, 490, 507, 562, 20, 0, 0, 0, 0, 0, 0, 0, 550, 461, 468, 510, 482, 515, 549, 500, 2, 2, 0, 0, 0, 0, 0, 0, 474, 497, 471, 541, 515, 575, 523, 466, 9, 0, 0, 0, 0, 0, 0, 0, 491, 527, 514, 493, 499, 519, 497, 512, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4159, 4167, 4059, 4114, 4039, 4071, 4069] avg=4096.0000 max=4167 max_rank=2 min=4039 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 517, 550, 503, 475, 521, 556, 473, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 518, 523, 519, 524, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 494, 513, 510, 491, 483, 523, 498, 497, 4, 2, 0, 0, 0, 0, 0, 0, 515, 529, 533, 484, 498, 501, 528, 465, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 494, 518, 505, 528, 517, 512, 20, 0, 0, 0, 0, 0, 0, 0, 484, 535, 526, 548, 514, 486, 507, 515, 2, 2, 0, 0, 0, 0, 0, 0, 509, 556, 510, 491, 512, 537, 491, 500, 9, 0, 0, 0, 0, 0, 0, 0, 523, 543, 568, 484, 506, 511, 510, 496, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4080, 4015, 4053, 4088, 4119, 4115, 4158] avg=4096.0000 max=4158 max_rank=7 min=4015 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 527, 530, 467, 497, 525, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0, 504, 519, 519, 474, 504, 535, 485, 517, 0, 0, 0, 0, 0, 0, 0, 0, 500, 505, 531, 494, 533, 491, 498, 503, 4, 2, 0, 0, 0, 0, 0, 0, 520, 556, 511, 519, 520, 520, 571, 464, 0, 0, 0, 0, 0, 0, 0, 0, 462, 497, 538, 523, 525, 505, 510, 498, 20, 0, 0, 0, 0, 0, 0, 0, 453, 529, 522, 472, 524, 494, 530, 507, 2, 1, 0, 0, 0, 0, 0, 0, 512, 526, 531, 475, 560, 514, 501, 500, 9, 0, 0, 0, 0, 0, 0, 0, 519, 543, 517, 534, 488, 524, 491, 471, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4057, 4061, 4181, 4078, 4034, 4128, 4104] avg=4096.0000 max=4181 max_rank=3 min=4034 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 477, 492, 484, 517, 534, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 515, 513, 519, 494, 471, 510, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 513, 500, 520, 554, 525, 485, 4, 2, 0, 0, 0, 0, 0, 0, 503, 534, 502, 545, 475, 529, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 501, 502, 518, 484, 493, 493, 525, 507, 21, 0, 0, 0, 0, 0, 0, 0, 514, 515, 544, 490, 497, 494, 541, 491, 2, 2, 0, 0, 0, 0, 0, 0, 538, 520, 544, 472, 491, 555, 499, 510, 9, 0, 0, 0, 0, 0, 0, 0, 549, 522, 509, 508, 540, 510, 509, 515, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4047, 4140, 4079, 4044, 4090, 4138, 4180] avg=4096.0000 max=4180 max_rank=7 min=4044 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[503, 522, 507, 564, 504, 493, 521, 474, 0, 0, 0, 0, 0, 0, 0, 0, 504, 489, 478, 547, 502, 514, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 493, 517, 502, 487, 505, 513, 4, 2, 0, 0, 0, 0, 0, 0, 505, 519, 513, 506, 520, 518, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 502, 539, 503, 519, 502, 548, 505, 520, 21, 0, 0, 0, 0, 0, 0, 0, 495, 484, 517, 521, 519, 496, 529, 515, 2, 2, 0, 0, 0, 0, 0, 0, 521, 499, 526, 470, 501, 489, 538, 540, 9, 0, 0, 0, 0, 0, 0, 0, 521, 533, 515, 502, 513, 545, 517, 494, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4027, 4069, 4094, 4159, 4080, 4093, 4158] avg=4096.0000 max=4159 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 481, 490, 493, 498, 481, 538, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 486, 532, 535, 542, 499, 496, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 543, 517, 479, 522, 519, 502, 4, 2, 0, 0, 0, 0, 0, 0, 517, 530, 494, 542, 511, 521, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 468, 554, 485, 496, 532, 480, 22, 0, 0, 0, 0, 0, 0, 0, 521, 539, 516, 511, 560, 524, 498, 528, 2, 2, 0, 0, 0, 0, 0, 0, 514, 481, 530, 533, 523, 475, 529, 496, 10, 0, 0, 0, 0, 0, 0, 0, 478, 482, 546, 461, 515, 492, 538, 510, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4132, 4115, 4140, 4048, 4201, 4091, 4041] avg=4096.0000 max=4201 max_rank=5 min=4000 min_rank=0 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[529, 574, 495, 510, 531, 526, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 551, 524, 508, 500, 527, 507, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 513, 521, 490, 488, 503, 505, 4, 2, 0, 0, 0, 0, 0, 0, 523, 522, 503, 504, 538, 495, 485, 452, 0, 0, 0, 0, 0, 0, 0, 0, 483, 512, 488, 503, 512, 542, 529, 521, 19, 0, 0, 0, 0, 0, 0, 0, 546, 489, 521, 502, 491, 508, 523, 514, 2, 2, 0, 0, 0, 0, 0, 0, 503, 530, 522, 520, 486, 502, 477, 512, 8, 0, 0, 0, 0, 0, 0, 0, 533, 493, 514, 508, 518, 521, 447, 514, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4157, 4062, 4022, 4109, 4098, 4060, 4064] avg=4096.0000 max=4196 max_rank=0 min=4022 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 495, 526, 498, 521, 513, 492, 0, 0, 0, 0, 0, 0, 0, 0, 512, 534, 504, 496, 530, 503, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 487, 553, 537, 520, 505, 497, 9, 8, 0, 0, 0, 0, 0, 0, 494, 534, 520, 516, 508, 529, 518, 521, 0, 0, 0, 0, 0, 0, 0, 0, 555, 508, 518, 490, 521, 482, 526, 501, 27, 0, 0, 0, 0, 0, 0, 0, 515, 513, 531, 529, 501, 521, 503, 484, 6, 4, 0, 0, 0, 0, 0, 0, 484, 501, 514, 518, 506, 502, 535, 498, 14, 0, 0, 0, 0, 0, 0, 0, 518, 495, 458, 508, 492, 484, 497, 518, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4109, 4139, 4140, 4128, 4107, 4072, 3994] avg=4096.0000 max=4140 max_rank=3 min=3994 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4352, 4096, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[491, 540, 508, 508, 516, 502, 513, 522, 497, 539, 532, 531, 514, 552, 546, 469, 516, 517, 525, 504, 508, 508, 496, 524, 518, 523, 513, 542, 475, 468, 503, 545, 513, 506, 521, 520, 520, 508, 486, 483, 472, 519, 532, 490, 538, 534, 505, 505, 507, 520, 529, 494, 532, 507, 507, 498, 562, 518, 508, 501, 470, 519, 487, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[513, 512, 492, 526, 524, 525, 499, 516, 520, 489, 509, 507, 503, 544, 514, 483, 519, 544, 497, 485, 547, 519, 513, 468, 531, 528, 479, 510, 512, 536, 540, 504, 494, 542, 563, 536, 502, 506, 507, 520, 526, 489, 504, 494, 529, 500, 494, 500, 495, 510, 504, 492, 510, 515, 545, 511, 500, 551, 480, 505, 524, 482, 508, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[469, 498, 483, 473, 493, 542, 511, 518, 511, 525, 540, 511, 496, 512, 497, 491, 541, 501, 499, 536, 521, 547, 506, 537, 494, 533, 547, 541, 537, 500, 510, 492, 520, 529, 505, 553, 505, 491, 501, 517, 504, 483, 500, 511, 581, 515, 529, 519, 472, 516, 476, 547, 521, 502, 516, 487, 509, 510, 502, 528, 493, 490, 489, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[501, 546, 469, 520, 487, 499, 489, 512, 537, 498, 497, 511, 542, 505, 504, 488, 514, 555, 476, 507, 528, 506, 518, 500, 521, 535, 498, 519, 498, 511, 469, 502, 524, 553, 464, 500, 526, 519, 521, 512, 507, 511, 503, 501, 528, 485, 516, 528, 486, 511, 511, 520, 532, 520, 540, 510, 511, 545, 538, 524, 500, 513, 541, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[471, 558, 539, 538, 515, 509, 541, 508, 512, 541, 495, 487, 491, 544, 544, 477, 492, 494, 485, 502, 505, 499, 516, 511, 532, 523, 486, 486, 497, 556, 500, 523, 498, 515, 524, 482, 504, 499, 531, 499, 531, 509, 512, 528, 533, 492, 531, 529, 522, 548, 502, 466, 528, 488, 514, 486, 528, 529, 511, 509, 569, 496, 490, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[560, 512, 515, 487, 523, 493, 499, 500, 507, 540, 485, 505, 505, 504, 499, 516, 511, 495, 517, 527, 536, 498, 517, 520, 498, 508, 532, 511, 548, 552, 515, 533, 531, 462, 501, 539, 545, 476, 476, 521, 508, 468, 535, 512, 518, 515, 522, 463, 516, 507, 502, 519, 529, 507, 528, 509, 500, 497, 496, 528, 539, 523, 523, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[518, 528, 549, 538, 530, 491, 509, 496, 524, 496, 491, 495, 535, 503, 508, 562, 498, 529, 488, 516, 521, 523, 498, 524, 519, 517, 525, 497, 487, 505, 525, 518, 495, 518, 520, 517, 484, 490, 518, 521, 521, 510, 476, 534, 494, 541, 530, 521, 502, 498, 503, 540, 486, 492, 542, 548, 510, 511, 478, 492, 486, 539, 500, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[504, 481, 505, 503, 497, 489, 536, 505, 516, 558, 525, 529, 515, 511, 523, 511, 533, 519, 503, 526, 538, 525, 546, 497, 535, 543, 470, 499, 537, 500, 505, 549, 498, 503, 543, 496, 524, 500, 455, 531, 495, 520, 515, 523, 513, 494, 487, 493, 518, 504, 479, 532, 520, 540, 544, 496, 491, 482, 488, 504, 519, 530, 493, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[471, 558, 539, 538, 515, 509, 541, 508, 10, 2, 0, 0, 0, 0, 0, 0, 512, 529, 495, 487, 491, 544, 544, 477, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 485, 502, 491, 499, 516, 511, 0, 0, 0, 0, 0, 0, 0, 0, 532, 499, 486, 486, 497, 556, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 524, 482, 504, 499, 531, 499, 6, 4, 0, 0, 0, 0, 0, 0, 531, 509, 512, 528, 533, 492, 531, 529, 5, 0, 0, 0, 0, 0, 0, 0, 522, 548, 502, 466, 528, 488, 514, 486, 1, 0, 0, 0, 0, 0, 0, 0, 528, 529, 511, 509, 569, 496, 490, 488, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4079, 3990, 4079, 4062, 4170, 4055, 4142] avg=4096.0000 max=4191 max_rank=0 min=3990 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[501, 546, 469, 520, 487, 499, 489, 512, 10, 2, 0, 0, 0, 0, 0, 0, 537, 487, 497, 511, 542, 505, 504, 488, 0, 0, 0, 0, 0, 0, 0, 0, 514, 555, 476, 507, 514, 506, 518, 500, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 498, 519, 498, 511, 469, 502, 0, 0, 0, 0, 0, 0, 0, 0, 524, 553, 464, 500, 526, 519, 521, 512, 5, 4, 0, 0, 0, 0, 0, 0, 507, 511, 503, 501, 528, 485, 516, 528, 5, 0, 0, 0, 0, 0, 0, 0, 486, 511, 511, 520, 532, 520, 540, 510, 1, 0, 0, 0, 0, 0, 0, 0, 511, 545, 538, 524, 500, 513, 541, 506, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4071, 4090, 4029, 4128, 4084, 4131, 4200] avg=4096.0000 max=4200 max_rank=7 min=4029 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[491, 540, 508, 508, 516, 502, 513, 522, 10, 2, 0, 0, 0, 0, 0, 0, 497, 527, 532, 531, 514, 552, 546, 469, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 525, 504, 494, 508, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 513, 542, 475, 468, 503, 545, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 521, 520, 520, 508, 486, 483, 6, 4, 0, 0, 0, 0, 0, 0, 472, 519, 532, 490, 538, 534, 505, 505, 5, 0, 0, 0, 0, 0, 0, 0, 507, 520, 529, 494, 532, 507, 507, 498, 1, 0, 0, 0, 0, 0, 0, 0, 562, 518, 508, 501, 470, 519, 487, 492, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4168, 4084, 4063, 4067, 4100, 4095, 4079] avg=4096.0000 max=4168 max_rank=1 min=4063 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 512, 492, 526, 524, 525, 499, 516, 11, 2, 0, 0, 0, 0, 0, 0, 520, 478, 509, 507, 503, 544, 514, 483, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 497, 485, 532, 519, 513, 468, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 479, 510, 512, 536, 540, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 563, 536, 502, 506, 507, 520, 5, 4, 0, 0, 0, 0, 0, 0, 526, 489, 504, 494, 529, 500, 494, 500, 5, 0, 0, 0, 0, 0, 0, 0, 495, 510, 504, 492, 510, 515, 545, 511, 1, 0, 0, 0, 0, 0, 0, 0, 500, 551, 480, 505, 524, 482, 508, 522, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4058, 4077, 4116, 4179, 4041, 4083, 4094] avg=4096.0000 max=4179 max_rank=4 min=4041 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[560, 512, 515, 487, 523, 493, 499, 500, 10, 2, 0, 0, 0, 0, 0, 0, 507, 528, 485, 505, 505, 504, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 517, 527, 522, 498, 517, 520, 0, 0, 0, 0, 0, 0, 0, 0, 498, 485, 532, 511, 548, 552, 515, 533, 0, 0, 0, 0, 0, 0, 0, 0, 531, 462, 501, 539, 545, 476, 476, 521, 6, 4, 0, 0, 0, 0, 0, 0, 508, 468, 535, 512, 518, 515, 522, 463, 5, 0, 0, 0, 0, 0, 0, 0, 516, 507, 502, 519, 529, 507, 528, 509, 1, 0, 0, 0, 0, 0, 0, 0, 500, 497, 496, 528, 539, 523, 523, 485, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4049, 4107, 4174, 4061, 4046, 4118, 4112] avg=4096.0000 max=4174 max_rank=3 min=4046 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[469, 498, 483, 473, 493, 542, 511, 518, 14, 5, 0, 0, 0, 0, 0, 0, 511, 506, 540, 511, 496, 512, 497, 491, 0, 0, 0, 0, 0, 0, 0, 0, 541, 501, 499, 536, 497, 547, 506, 537, 0, 0, 0, 0, 0, 0, 0, 0, 494, 502, 547, 541, 537, 500, 510, 492, 0, 0, 0, 0, 0, 0, 0, 0, 520, 529, 505, 553, 505, 491, 501, 517, 8, 10, 0, 0, 0, 0, 0, 0, 504, 483, 500, 511, 581, 515, 529, 519, 8, 0, 0, 0, 0, 0, 0, 0, 472, 516, 476, 547, 521, 502, 516, 487, 3, 0, 0, 0, 0, 0, 0, 0, 509, 510, 502, 528, 493, 490, 489, 535, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4064, 4164, 4123, 4139, 4150, 4040, 4082] avg=4096.0000 max=4164 max_rank=2 min=4006 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 528, 549, 538, 530, 491, 509, 496, 10, 2, 0, 0, 0, 0, 0, 0, 524, 485, 491, 495, 535, 503, 508, 562, 0, 0, 0, 0, 0, 0, 0, 0, 498, 529, 488, 516, 507, 523, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 525, 497, 487, 505, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 520, 517, 484, 490, 518, 521, 5, 4, 0, 0, 0, 0, 0, 0, 521, 510, 476, 534, 494, 541, 530, 521, 5, 0, 0, 0, 0, 0, 0, 0, 502, 498, 503, 540, 486, 492, 542, 548, 1, 0, 0, 0, 0, 0, 0, 0, 510, 511, 478, 492, 486, 539, 500, 488, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4103, 4083, 4070, 4072, 4132, 4112, 4025] avg=4096.0000 max=4171 max_rank=0 min=4025 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[504, 481, 505, 503, 497, 489, 536, 505, 10, 2, 0, 0, 0, 0, 0, 0, 516, 546, 525, 529, 515, 511, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 519, 503, 526, 524, 525, 546, 497, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 470, 499, 537, 500, 505, 549, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 543, 496, 524, 500, 455, 531, 6, 4, 0, 0, 0, 0, 0, 0, 495, 520, 515, 523, 513, 494, 487, 493, 5, 0, 0, 0, 0, 0, 0, 0, 518, 504, 479, 532, 520, 540, 544, 496, 1, 0, 0, 0, 0, 0, 0, 0, 491, 482, 488, 504, 519, 530, 493, 505, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4176, 4173, 4114, 4060, 4045, 4134, 4034] avg=4096.0000 max=4176 max_rank=1 min=4032 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4352, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[500, 483, 533, 519, 520, 534, 510, 525, 482, 561, 552, 506, 540, 519, 495, 512, 515, 522, 544, 513, 505, 503, 483, 538, 572, 523, 531, 511, 510, 502, 520, 524, 509, 512, 504, 522, 453, 538, 499, 511, 494, 498, 518, 496, 516, 536, 508, 510, 499, 489, 480, 517, 511, 513, 534, 482, 492, 499, 498, 531, 514, 514, 505, 459] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[499, 519, 482, 501, 506, 501, 551, 508, 527, 504, 505, 536, 538, 511, 525, 513, 510, 512, 519, 527, 527, 496, 531, 506, 499, 509, 509, 507, 529, 491, 521, 489, 505, 508, 471, 539, 506, 499, 545, 505, 541, 534, 514, 552, 486, 498, 504, 491, 511, 535, 510, 468, 509, 550, 498, 498, 494, 499, 480, 517, 525, 547, 520, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[547, 495, 512, 528, 487, 500, 492, 535, 528, 508, 487, 472, 516, 551, 528, 514, 514, 506, 460, 521, 537, 499, 504, 559, 485, 533, 492, 511, 509, 496, 484, 525, 521, 510, 524, 497, 541, 528, 490, 509, 511, 504, 519, 518, 561, 480, 510, 491, 484, 538, 528, 525, 522, 513, 538, 513, 474, 520, 450, 563, 517, 478, 526, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[494, 511, 480, 504, 490, 519, 543, 551, 523, 525, 522, 541, 515, 528, 491, 522, 456, 473, 497, 523, 530, 490, 494, 465, 504, 541, 496, 539, 506, 464, 508, 514, 545, 506, 505, 515, 484, 502, 499, 530, 557, 530, 514, 518, 532, 530, 550, 526, 529, 501, 505, 499, 515, 531, 534, 499, 537, 514, 501, 504, 456, 541, 489, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[472, 498, 526, 531, 515, 505, 567, 503, 503, 524, 508, 515, 525, 484, 509, 528, 530, 554, 496, 513, 456, 539, 478, 514, 542, 505, 481, 506, 530, 503, 504, 535, 575, 539, 533, 506, 506, 503, 560, 511, 523, 479, 508, 468, 512, 529, 527, 476, 514, 510, 500, 515, 497, 506, 541, 540, 500, 518, 499, 515, 481, 500, 483, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[514, 506, 542, 529, 515, 510, 517, 509, 499, 507, 466, 512, 494, 498, 512, 505, 518, 510, 483, 525, 534, 524, 523, 512, 525, 520, 534, 509, 543, 508, 523, 513, 469, 515, 519, 461, 529, 496, 499, 526, 511, 560, 505, 500, 523, 523, 523, 510, 527, 513, 520, 494, 494, 516, 479, 533, 516, 516, 489, 494, 518, 518, 497, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[512, 526, 546, 519, 524, 527, 522, 532, 510, 504, 506, 488, 516, 511, 472, 468, 488, 522, 527, 537, 536, 513, 486, 526, 522, 531, 533, 513, 511, 518, 542, 532, 559, 479, 464, 551, 497, 498, 528, 492, 473, 475, 497, 503, 487, 528, 513, 523, 492, 498, 498, 547, 507, 476, 486, 514, 530, 513, 482, 534, 511, 505, 548, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[517, 523, 498, 513, 496, 503, 528, 534, 517, 519, 508, 528, 511, 516, 515, 490, 510, 529, 522, 549, 503, 514, 522, 534, 545, 503, 496, 512, 559, 513, 518, 516, 499, 502, 498, 479, 492, 522, 517, 525, 525, 521, 490, 507, 500, 516, 524, 516, 493, 533, 514, 525, 530, 501, 499, 483, 507, 464, 508, 481, 507, 517, 503, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 483, 533, 519, 520, 534, 488, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 561, 552, 506, 540, 519, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 544, 509, 505, 503, 483, 538, 0, 0, 0, 0, 0, 0, 0, 0, 572, 523, 531, 511, 482, 502, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 504, 522, 453, 538, 499, 511, 6, 4, 0, 0, 0, 0, 0, 0, 494, 498, 518, 496, 516, 536, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 480, 517, 511, 513, 534, 482, 11, 0, 0, 0, 0, 0, 0, 0, 492, 499, 498, 531, 514, 514, 505, 459, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4167, 4119, 4165, 4058, 4070, 4036, 4051] avg=4096.0000 max=4167 max_rank=1 min=4036 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 519, 482, 501, 506, 501, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 505, 536, 538, 511, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 510, 512, 519, 523, 527, 496, 531, 506, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 509, 507, 500, 491, 521, 489, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 471, 539, 506, 499, 545, 505, 6, 4, 0, 0, 0, 0, 0, 0, 541, 534, 514, 552, 486, 498, 498, 491, 0, 0, 0, 0, 0, 0, 0, 0, 511, 535, 510, 468, 509, 550, 498, 498, 12, 0, 0, 0, 0, 0, 0, 0, 494, 499, 480, 517, 525, 547, 520, 501, 29, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4159, 4124, 4025, 4088, 4114, 4091, 4124] avg=4096.0000 max=4159 max_rank=1 min=4025 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[547, 495, 512, 528, 487, 500, 471, 535, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 487, 472, 516, 551, 528, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 460, 517, 537, 499, 504, 559, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 492, 511, 481, 496, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 524, 497, 541, 528, 490, 509, 6, 4, 0, 0, 0, 0, 0, 0, 511, 504, 519, 518, 561, 480, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 484, 538, 528, 525, 522, 513, 538, 513, 10, 0, 0, 0, 0, 0, 0, 0, 474, 520, 450, 563, 517, 478, 526, 530, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4104, 4096, 4007, 4130, 4088, 4171, 4097] avg=4096.0000 max=4171 max_rank=6 min=4007 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 506, 542, 529, 515, 510, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 499, 507, 466, 512, 494, 498, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 483, 521, 534, 524, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 525, 520, 534, 509, 514, 508, 523, 513, 0, 0, 0, 0, 0, 0, 0, 0, 469, 515, 519, 461, 529, 496, 499, 526, 6, 4, 0, 0, 0, 0, 0, 0, 511, 560, 505, 500, 523, 523, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 520, 494, 494, 516, 479, 533, 11, 0, 0, 0, 0, 0, 0, 0, 516, 516, 489, 494, 518, 518, 497, 536, 29, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 3993, 4125, 4146, 4024, 4149, 4087, 4124] avg=4096.0000 max=4149 max_rank=5 min=3993 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[494, 511, 480, 504, 490, 519, 512, 551, 0, 0, 0, 0, 0, 0, 0, 0, 523, 525, 522, 541, 515, 528, 491, 522, 3, 0, 0, 0, 0, 0, 0, 0, 456, 473, 497, 513, 530, 490, 494, 465, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 496, 539, 476, 464, 508, 514, 0, 0, 0, 0, 0, 0, 0, 0, 545, 506, 505, 515, 484, 502, 499, 530, 10, 7, 0, 0, 0, 0, 0, 0, 557, 530, 514, 518, 532, 530, 538, 526, 0, 0, 0, 0, 0, 0, 0, 0, 529, 501, 505, 499, 515, 531, 534, 499, 15, 2, 0, 0, 0, 0, 0, 0, 537, 514, 501, 504, 456, 541, 489, 511, 30, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4170, 3918, 4042, 4103, 4245, 4130, 4099] avg=4096.0000 max=4245 max_rank=5 min=3918 min_rank=2 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16577 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[472, 498, 526, 531, 515, 505, 542, 503, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 508, 515, 525, 484, 509, 528, 0, 0, 0, 0, 0, 0, 0, 0, 530, 554, 496, 509, 456, 539, 478, 514, 0, 0, 0, 0, 0, 0, 0, 0, 542, 505, 481, 506, 501, 503, 504, 535, 0, 0, 0, 0, 0, 0, 0, 0, 575, 539, 533, 506, 506, 503, 560, 511, 6, 4, 0, 0, 0, 0, 0, 0, 523, 479, 508, 468, 512, 529, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 500, 515, 497, 506, 541, 540, 12, 0, 0, 0, 0, 0, 0, 0, 500, 518, 499, 515, 481, 500, 483, 495, 29, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4076, 4077, 4243, 4016, 4135, 4033] avg=4096.0000 max=4243 max_rank=4 min=4016 min_rank=5 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 526, 546, 519, 524, 527, 500, 532, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 506, 488, 516, 511, 472, 468, 0, 0, 0, 0, 0, 0, 0, 0, 488, 522, 527, 533, 536, 513, 486, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 533, 513, 483, 518, 542, 532, 0, 0, 0, 0, 0, 0, 0, 0, 559, 479, 464, 551, 497, 498, 528, 492, 6, 4, 0, 0, 0, 0, 0, 0, 473, 475, 497, 503, 487, 528, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 492, 498, 498, 547, 507, 476, 486, 514, 11, 0, 0, 0, 0, 0, 0, 0, 530, 513, 482, 534, 511, 505, 548, 540, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 3975, 4131, 4174, 4078, 3993, 4029, 4202] avg=4096.0000 max=4202 max_rank=7 min=3975 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 523, 498, 513, 496, 503, 505, 534, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 508, 528, 511, 516, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 510, 529, 522, 545, 503, 514, 522, 534, 0, 0, 0, 0, 0, 0, 0, 0, 545, 503, 496, 512, 529, 513, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 502, 498, 479, 492, 522, 517, 525, 6, 4, 0, 0, 0, 0, 0, 0, 525, 521, 490, 507, 500, 516, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 493, 533, 514, 525, 530, 501, 499, 483, 11, 0, 0, 0, 0, 0, 0, 0, 507, 464, 508, 481, 507, 517, 503, 509, 30, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4104, 4179, 4132, 4044, 4093, 4089, 4038] avg=4096.0000 max=4179 max_rank=2 min=4038 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[503, 520, 540, 495, 541, 541, 516, 498, 500, 510, 506, 527, 517, 510, 530, 545, 533, 492, 515, 503, 476, 552, 561, 517, 520, 526, 540, 498, 494, 513, 489, 517, 502, 479, 533, 511, 489, 490, 507, 470, 501, 541, 517, 522, 500, 506, 493, 548, 503, 491, 482, 490, 495, 526, 494, 526, 486, 477, 562, 491, 501, 519, 519, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[519, 513, 505, 517, 523, 510, 515, 507, 527, 502, 510, 512, 461, 508, 513, 533, 516, 520, 482, 539, 493, 513, 548, 522, 495, 478, 495, 538, 509, 502, 527, 476, 508, 546, 530, 510, 528, 520, 491, 507, 505, 479, 513, 486, 534, 513, 557, 523, 482, 529, 494, 527, 511, 570, 523, 501, 476, 542, 479, 494, 534, 485, 516, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[523, 511, 499, 522, 530, 567, 527, 516, 531, 462, 495, 533, 488, 479, 470, 506, 527, 510, 497, 496, 524, 490, 501, 518, 480, 519, 489, 505, 502, 476, 530, 550, 485, 526, 559, 499, 554, 527, 508, 500, 488, 483, 519, 533, 531, 531, 514, 507, 490, 532, 512, 501, 534, 468, 518, 518, 500, 503, 492, 503, 564, 537, 530, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[468, 476, 497, 497, 520, 500, 508, 523, 513, 531, 529, 527, 517, 497, 519, 514, 496, 503, 546, 513, 499, 518, 503, 493, 543, 519, 510, 476, 504, 507, 549, 484, 488, 490, 521, 528, 523, 514, 515, 495, 501, 529, 520, 493, 493, 514, 553, 483, 506, 499, 510, 553, 492, 514, 528, 550, 533, 497, 481, 542, 543, 523, 514, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[521, 538, 466, 527, 546, 502, 553, 489, 537, 500, 530, 508, 483, 515, 528, 490, 549, 483, 484, 491, 512, 531, 539, 529, 484, 502, 503, 493, 500, 508, 542, 520, 524, 519, 512, 524, 518, 508, 530, 527, 491, 508, 499, 502, 514, 516, 525, 528, 485, 495, 493, 475, 503, 517, 545, 519, 536, 503, 499, 521, 501, 519, 512, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[518, 501, 540, 544, 500, 530, 530, 512, 532, 513, 533, 532, 533, 523, 521, 512, 548, 500, 524, 517, 458, 502, 481, 550, 491, 512, 504, 523, 508, 500, 530, 493, 482, 512, 522, 519, 503, 484, 544, 475, 521, 495, 547, 532, 491, 475, 512, 520, 489, 495, 505, 557, 530, 478, 514, 506, 509, 511, 498, 541, 485, 495, 518, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[499, 515, 513, 509, 503, 504, 469, 528, 523, 532, 504, 513, 482, 509, 496, 542, 504, 509, 500, 509, 474, 491, 494, 523, 501, 534, 514, 456, 546, 490, 505, 521, 499, 511, 503, 539, 507, 499, 480, 534, 544, 552, 527, 520, 524, 511, 510, 523, 496, 519, 500, 489, 558, 530, 513, 564, 487, 522, 486, 508, 508, 523, 512, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[539, 523, 518, 500, 500, 523, 547, 540, 492, 494, 467, 538, 518, 519, 506, 530, 525, 535, 526, 511, 505, 523, 499, 533, 493, 523, 482, 457, 495, 517, 489, 531, 519, 483, 503, 502, 485, 501, 505, 516, 513, 509, 536, 518, 526, 488, 510, 491, 509, 550, 493, 520, 533, 536, 496, 531, 528, 547, 497, 512, 459, 562, 509, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[468, 476, 497, 497, 520, 473, 508, 523, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 529, 526, 517, 497, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 503, 546, 513, 499, 518, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 510, 476, 504, 507, 549, 484, 27, 14, 0, 0, 0, 0, 0, 0, 488, 490, 521, 528, 523, 514, 515, 495, 5, 5, 3, 1, 0, 0, 0, 0, 501, 529, 503, 493, 493, 514, 553, 483, 0, 0, 0, 0, 0, 0, 0, 0, 506, 499, 510, 553, 492, 514, 528, 545, 0, 0, 0, 0, 0, 0, 0, 0, 533, 497, 481, 542, 543, 518, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3962, 4146, 4071, 4133, 4088, 4069, 4147, 4152] avg=4096.0000 max=4152 max_rank=7 min=3962 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 538, 466, 527, 546, 475, 553, 489, 0, 0, 0, 0, 0, 0, 0, 0, 537, 500, 530, 507, 483, 515, 528, 490, 0, 0, 0, 0, 0, 0, 0, 0, 549, 483, 484, 491, 512, 531, 539, 529, 0, 0, 0, 0, 0, 0, 0, 0, 484, 502, 503, 493, 500, 508, 542, 520, 27, 13, 0, 0, 0, 0, 0, 0, 524, 519, 512, 524, 518, 508, 530, 527, 5, 5, 2, 1, 0, 0, 0, 0, 491, 508, 484, 502, 514, 516, 525, 528, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 493, 475, 503, 517, 545, 514, 0, 0, 0, 0, 0, 0, 0, 0, 536, 503, 499, 521, 501, 514, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4090, 4118, 4092, 4175, 4068, 4027, 4083] avg=4096.0000 max=4175 max_rank=4 min=4027 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 513, 505, 517, 523, 483, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 510, 511, 461, 508, 513, 533, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 482, 539, 493, 513, 548, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 478, 495, 538, 509, 502, 527, 476, 27, 14, 0, 0, 0, 0, 0, 0, 508, 546, 530, 510, 528, 520, 491, 507, 5, 4, 3, 1, 0, 0, 0, 0, 505, 479, 496, 486, 534, 513, 557, 523, 0, 0, 0, 0, 0, 0, 0, 0, 482, 529, 494, 527, 511, 570, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 476, 542, 479, 494, 534, 481, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4065, 4133, 4061, 4153, 4093, 4132, 4049] avg=4096.0000 max=4153 max_rank=4 min=4049 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 520, 540, 495, 541, 512, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 506, 526, 517, 510, 530, 545, 0, 0, 0, 0, 0, 0, 0, 0, 533, 492, 515, 503, 476, 552, 561, 517, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 540, 498, 494, 513, 489, 517, 29, 14, 0, 0, 0, 0, 0, 0, 502, 479, 533, 511, 489, 490, 507, 470, 5, 5, 3, 1, 0, 0, 0, 0, 501, 541, 500, 522, 500, 506, 493, 548, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 482, 490, 495, 526, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 486, 477, 562, 491, 501, 514, 519, 552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4144, 4149, 4140, 3995, 4111, 4002, 4102] avg=4096.0000 max=4149 max_rank=2 min=3995 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 511, 499, 522, 530, 537, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 531, 462, 495, 532, 488, 479, 470, 506, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 497, 496, 524, 490, 501, 518, 0, 0, 0, 0, 0, 0, 0, 0, 480, 519, 489, 505, 502, 476, 530, 550, 30, 14, 0, 0, 0, 0, 0, 0, 485, 526, 559, 499, 554, 527, 508, 500, 5, 5, 3, 1, 0, 0, 0, 0, 488, 483, 502, 533, 531, 531, 514, 507, 0, 0, 0, 0, 0, 0, 0, 0, 490, 532, 512, 501, 534, 468, 518, 513, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 492, 503, 564, 532, 530, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 3963, 4063, 4095, 4172, 4089, 4068, 4153] avg=4096.0000 max=4172 max_rank=4 min=3963 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 501, 540, 544, 500, 502, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 532, 513, 533, 531, 533, 523, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 548, 500, 524, 517, 458, 502, 481, 550, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 504, 523, 508, 500, 530, 493, 28, 15, 0, 0, 0, 0, 0, 0, 482, 512, 522, 519, 503, 484, 544, 475, 5, 4, 3, 1, 0, 0, 0, 0, 521, 495, 529, 532, 491, 475, 512, 520, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 505, 557, 530, 478, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 498, 541, 485, 491, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4198, 4080, 4104, 4054, 4075, 4069, 4041] avg=4096.0000 max=4198 max_rank=1 min=4041 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 515, 513, 509, 503, 477, 469, 528, 0, 0, 0, 0, 0, 0, 0, 0, 523, 532, 504, 512, 482, 509, 496, 542, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 500, 509, 474, 491, 494, 523, 0, 0, 0, 0, 0, 0, 0, 0, 501, 534, 514, 456, 546, 490, 505, 521, 27, 14, 0, 0, 0, 0, 0, 0, 499, 511, 503, 539, 507, 499, 480, 534, 5, 5, 3, 1, 0, 0, 0, 0, 544, 552, 510, 520, 524, 511, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 500, 489, 558, 530, 513, 559, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 486, 508, 508, 518, 512, 558, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4100, 4004, 4108, 4086, 4194, 4164, 4099] avg=4096.0000 max=4194 max_rank=5 min=4004 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[539, 523, 518, 500, 500, 492, 547, 540, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 467, 536, 518, 519, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 526, 511, 505, 523, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 482, 457, 495, 517, 489, 531, 31, 17, 0, 0, 0, 0, 0, 0, 519, 483, 503, 502, 485, 501, 505, 516, 9, 8, 5, 2, 7, 0, 0, 0, 513, 509, 514, 518, 526, 488, 510, 491, 0, 0, 0, 0, 0, 0, 0, 0, 509, 550, 493, 520, 533, 536, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 528, 547, 497, 512, 459, 554, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4062, 4150, 4035, 4045, 4069, 4159, 4089] avg=4096.0000 max=4159 max_rank=0 min=4035 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[482, 484, 543, 502, 499, 501, 519, 495, 507, 522, 496, 514, 496, 539, 522, 553, 541, 536, 533, 533, 555, 525, 537, 535, 514, 512, 470, 507, 519, 498, 505, 516, 506, 507, 516, 501, 507, 489, 473, 524, 471, 483, 512, 549, 521, 505, 503, 506, 515, 538, 529, 523, 508, 537, 514, 505, 519, 497, 520, 496, 474, 511, 494, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[484, 481, 500, 542, 497, 465, 547, 538, 465, 497, 539, 517, 526, 527, 509, 537, 511, 470, 524, 479, 538, 499, 508, 483, 499, 504, 502, 490, 470, 486, 521, 517, 492, 520, 475, 525, 503, 536, 502, 525, 491, 540, 515, 532, 505, 525, 546, 516, 540, 532, 499, 554, 538, 536, 494, 501, 555, 458, 513, 500, 564, 494, 539, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[471, 506, 495, 516, 493, 516, 515, 504, 524, 529, 488, 490, 530, 536, 474, 507, 501, 504, 494, 494, 493, 536, 556, 499, 509, 583, 481, 502, 543, 513, 512, 511, 533, 529, 530, 492, 554, 517, 545, 487, 489, 524, 511, 501, 512, 508, 528, 520, 485, 553, 492, 521, 498, 511, 535, 515, 535, 503, 534, 490, 489, 498, 458, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[535, 546, 482, 503, 501, 480, 541, 537, 530, 528, 508, 543, 508, 470, 499, 487, 525, 480, 486, 497, 506, 532, 531, 538, 528, 478, 492, 490, 498, 541, 496, 446, 516, 535, 515, 517, 534, 497, 504, 530, 497, 534, 530, 525, 515, 507, 505, 503, 500, 527, 523, 510, 513, 535, 536, 504, 507, 525, 503, 502, 522, 481, 577, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[500, 496, 485, 510, 512, 483, 503, 530, 520, 502, 494, 517, 520, 494, 532, 483, 540, 511, 530, 481, 486, 511, 556, 507, 497, 504, 530, 529, 508, 467, 549, 495, 518, 522, 532, 509, 523, 533, 530, 546, 538, 504, 495, 535, 539, 554, 503, 525, 478, 542, 514, 517, 473, 506, 520, 466, 535, 553, 506, 485, 481, 524, 512, 468] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[535, 529, 549, 492, 501, 528, 493, 519, 502, 522, 493, 520, 512, 501, 482, 511, 504, 510, 481, 512, 529, 466, 541, 508, 511, 514, 503, 477, 506, 517, 488, 517, 496, 529, 559, 540, 473, 504, 518, 543, 506, 530, 488, 520, 525, 501, 502, 492, 530, 495, 547, 555, 542, 498, 501, 510, 507, 545, 533, 474, 509, 515, 483, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[523, 529, 493, 529, 484, 497, 519, 524, 499, 512, 506, 516, 519, 533, 516, 527, 468, 528, 502, 528, 505, 539, 516, 498, 521, 493, 478, 494, 488, 526, 543, 520, 584, 528, 548, 521, 486, 511, 529, 517, 487, 537, 519, 504, 490, 473, 508, 539, 539, 558, 497, 457, 506, 512, 502, 509, 482, 534, 472, 533, 487, 524, 497, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[511, 457, 499, 515, 538, 521, 481, 496, 462, 493, 548, 529, 533, 467, 500, 522, 542, 479, 527, 506, 494, 519, 504, 543, 506, 490, 517, 499, 497, 500, 527, 516, 481, 531, 546, 509, 479, 511, 518, 541, 542, 532, 471, 505, 484, 487, 519, 498, 522, 509, 494, 543, 535, 554, 505, 477, 533, 525, 567, 517, 521, 500, 510, 564] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[482, 484, 543, 502, 499, 501, 519, 495, 33, 0, 0, 0, 0, 0, 0, 0, 507, 522, 496, 514, 496, 539, 522, 553, 8, 0, 0, 0, 0, 0, 0, 0, 541, 536, 533, 533, 555, 525, 528, 535, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 470, 507, 519, 498, 505, 516, 46, 0, 0, 0, 0, 0, 0, 0, 506, 507, 470, 501, 507, 489, 473, 524, 0, 0, 0, 0, 0, 0, 0, 0, 471, 482, 512, 549, 521, 505, 503, 506, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 529, 523, 508, 537, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 520, 496, 474, 511, 494, 505, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4157, 4286, 4087, 3977, 4049, 4136, 4018] avg=4096.0000 max=4286 max_rank=2 min=3977 min_rank=4 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 496, 485, 510, 512, 483, 503, 530, 33, 0, 0, 0, 0, 0, 0, 0, 520, 502, 494, 517, 520, 494, 532, 483, 8, 0, 0, 0, 0, 0, 0, 0, 540, 511, 530, 481, 486, 511, 547, 507, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 530, 529, 508, 467, 549, 495, 48, 0, 0, 0, 0, 0, 0, 0, 518, 522, 484, 509, 523, 533, 530, 546, 0, 0, 0, 0, 0, 0, 0, 0, 538, 503, 495, 535, 539, 554, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 478, 509, 514, 517, 473, 506, 520, 466, 0, 0, 0, 0, 0, 0, 0, 0, 535, 553, 506, 485, 481, 524, 512, 468, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4070, 4113, 4127, 4165, 4192, 3983, 4066] avg=4096.0000 max=4192 max_rank=5 min=3983 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[484, 481, 500, 542, 497, 465, 547, 538, 32, 0, 0, 0, 0, 0, 0, 0, 465, 497, 539, 517, 526, 527, 509, 537, 7, 0, 0, 0, 0, 0, 0, 0, 511, 470, 524, 479, 538, 499, 500, 483, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 502, 490, 470, 486, 521, 517, 43, 0, 0, 0, 0, 0, 0, 0, 492, 520, 432, 525, 503, 536, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 491, 539, 515, 532, 505, 525, 546, 516, 0, 0, 0, 0, 0, 0, 0, 0, 540, 500, 499, 554, 538, 536, 494, 501, 0, 0, 0, 0, 0, 0, 0, 0, 555, 458, 513, 500, 564, 494, 539, 531, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4124, 4004, 4032, 4035, 4169, 4162, 4156] avg=4096.0000 max=4169 max_rank=5 min=4004 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[471, 506, 495, 516, 493, 516, 515, 504, 33, 0, 0, 0, 0, 0, 0, 0, 524, 529, 488, 490, 530, 536, 474, 507, 8, 0, 0, 0, 0, 0, 0, 0, 501, 504, 494, 494, 493, 536, 547, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 583, 481, 502, 543, 513, 512, 511, 48, 0, 0, 0, 0, 0, 0, 0, 533, 529, 482, 492, 554, 517, 545, 487, 0, 0, 0, 0, 0, 0, 0, 0, 489, 523, 511, 501, 512, 508, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 492, 521, 498, 511, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 534, 490, 489, 498, 458, 546, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4086, 4068, 4202, 4139, 4092, 4077, 4055] avg=4096.0000 max=4202 max_rank=3 min=4049 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[535, 529, 549, 492, 501, 528, 493, 519, 33, 6, 0, 0, 0, 0, 0, 0, 502, 522, 493, 520, 512, 501, 482, 511, 12, 0, 0, 0, 0, 0, 0, 0, 504, 510, 481, 512, 529, 466, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 503, 477, 506, 517, 488, 517, 54, 5, 0, 0, 0, 0, 0, 0, 496, 529, 505, 540, 473, 504, 518, 543, 0, 0, 0, 0, 0, 0, 0, 0, 506, 524, 488, 520, 525, 501, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 530, 457, 547, 555, 542, 498, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 507, 545, 533, 474, 509, 515, 483, 525, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4055, 4028, 4092, 4108, 4058, 4140, 4102] avg=4096.0000 max=4185 max_rank=0 min=4028 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[535, 546, 482, 503, 501, 480, 541, 537, 32, 0, 0, 0, 0, 0, 0, 0, 530, 528, 508, 543, 508, 470, 499, 487, 7, 0, 0, 0, 0, 0, 0, 0, 525, 480, 486, 497, 506, 532, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 528, 478, 492, 490, 498, 541, 496, 446, 46, 0, 0, 0, 0, 0, 0, 0, 516, 535, 469, 517, 534, 497, 504, 530, 0, 0, 0, 0, 0, 0, 0, 0, 497, 533, 530, 525, 515, 507, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 523, 510, 513, 535, 536, 504, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 503, 502, 522, 481, 577, 477, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4080, 4087, 4015, 4102, 4115, 4116, 4096] avg=4096.0000 max=4157 max_rank=0 min=4015 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 529, 493, 529, 484, 497, 519, 524, 34, 0, 0, 0, 0, 0, 0, 0, 499, 512, 506, 516, 519, 533, 516, 527, 7, 0, 0, 0, 0, 0, 0, 0, 468, 528, 502, 528, 505, 539, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 521, 493, 478, 494, 488, 526, 543, 520, 49, 0, 0, 0, 0, 0, 0, 0, 584, 528, 499, 521, 486, 511, 529, 517, 0, 0, 0, 0, 0, 0, 0, 0, 487, 536, 519, 504, 490, 473, 508, 539, 0, 0, 0, 0, 0, 0, 0, 0, 539, 524, 497, 457, 506, 512, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 482, 534, 472, 533, 487, 524, 497, 505, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4135, 4076, 4112, 4175, 4056, 4046, 4036] avg=4096.0000 max=4175 max_rank=4 min=4036 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 457, 499, 515, 538, 521, 481, 496, 31, 0, 0, 0, 0, 0, 0, 0, 462, 493, 548, 529, 533, 467, 500, 522, 7, 0, 0, 0, 0, 0, 0, 0, 542, 479, 527, 506, 494, 519, 496, 543, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 517, 499, 497, 500, 527, 516, 49, 0, 0, 0, 0, 0, 0, 0, 481, 531, 497, 509, 479, 511, 518, 541, 0, 0, 0, 0, 0, 0, 0, 0, 542, 531, 471, 505, 484, 487, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 478, 494, 543, 535, 554, 505, 477, 0, 0, 0, 0, 0, 0, 0, 0, 533, 525, 567, 517, 521, 500, 510, 564, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4061, 4106, 4101, 4067, 4037, 4108, 4239] avg=4096.0000 max=4239 max_rank=7 min=4037 min_rank=5 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3840, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[542, 530, 488, 500, 474, 497, 519, 519, 514, 494, 504, 506, 529, 512, 501, 498, 519, 538, 555, 521, 564, 457, 508, 497, 485, 530, 527, 529, 508, 501, 537, 546, 501, 528, 546, 491, 529, 457, 506, 501, 529, 530, 531, 489, 504, 485, 509, 448, 518, 451, 490, 528, 513, 556, 527, 538, 524, 509, 489, 520, 496, 523, 530, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[539, 518, 526, 491, 539, 481, 488, 528, 487, 523, 486, 505, 492, 510, 482, 477, 527, 503, 526, 519, 515, 490, 500, 542, 536, 546, 525, 503, 482, 528, 549, 477, 503, 521, 498, 497, 495, 537, 516, 556, 503, 518, 523, 537, 471, 521, 547, 523, 472, 543, 494, 536, 514, 539, 487, 501, 548, 474, 476, 500, 485, 513, 535, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[512, 540, 546, 497, 519, 520, 511, 482, 514, 514, 489, 511, 517, 479, 525, 510, 532, 470, 543, 488, 483, 486, 515, 498, 544, 535, 575, 514, 510, 542, 500, 487, 521, 500, 490, 526, 488, 480, 490, 538, 505, 509, 532, 511, 532, 497, 527, 528, 516, 530, 502, 516, 536, 514, 506, 462, 489, 549, 507, 499, 533, 510, 544, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[509, 505, 501, 488, 469, 467, 530, 537, 534, 487, 541, 497, 513, 472, 521, 492, 468, 555, 490, 508, 517, 533, 508, 542, 523, 484, 511, 513, 514, 505, 527, 533, 546, 507, 490, 518, 522, 518, 488, 500, 473, 518, 525, 485, 511, 513, 525, 499, 535, 517, 508, 530, 505, 532, 541, 537, 509, 505, 505, 542, 526, 501, 526, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[528, 475, 516, 544, 516, 533, 504, 507, 513, 500, 485, 522, 508, 476, 507, 498, 530, 515, 525, 503, 503, 493, 517, 532, 500, 513, 484, 530, 501, 492, 511, 503, 540, 482, 503, 496, 533, 543, 498, 518, 501, 540, 466, 527, 511, 517, 531, 498, 477, 539, 491, 502, 500, 547, 544, 547, 529, 524, 511, 487, 509, 506, 529, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[504, 490, 510, 539, 497, 507, 506, 486, 511, 504, 548, 496, 480, 484, 501, 545, 502, 535, 507, 504, 559, 528, 510, 516, 526, 499, 485, 510, 537, 517, 508, 480, 481, 531, 519, 509, 548, 522, 518, 513, 482, 506, 499, 546, 563, 486, 537, 483, 512, 525, 511, 538, 490, 524, 530, 500, 498, 541, 521, 517, 488, 466, 509, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[564, 535, 466, 488, 490, 500, 482, 520, 538, 496, 509, 524, 495, 503, 524, 510, 517, 512, 520, 507, 546, 507, 478, 530, 520, 520, 521, 540, 492, 502, 512, 498, 474, 517, 545, 526, 540, 500, 500, 501, 503, 533, 523, 504, 516, 509, 538, 465, 515, 497, 536, 526, 514, 525, 527, 493, 509, 543, 527, 517, 500, 478, 487, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[516, 513, 511, 484, 552, 473, 470, 527, 486, 539, 508, 501, 511, 547, 505, 492, 503, 496, 538, 512, 511, 518, 506, 484, 509, 521, 522, 506, 497, 511, 494, 491, 504, 540, 515, 483, 531, 473, 526, 497, 556, 514, 539, 497, 527, 529, 502, 540, 506, 486, 582, 492, 521, 531, 535, 466, 498, 535, 506, 509, 530, 494, 518, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[542, 530, 488, 500, 474, 497, 519, 519, 13, 9, 1, 0, 0, 0, 0, 0, 514, 494, 504, 506, 529, 512, 501, 498, 33, 14, 0, 0, 0, 0, 0, 0, 519, 538, 541, 521, 564, 457, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 485, 530, 514, 529, 508, 501, 537, 546, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 546, 491, 529, 457, 506, 501, 4, 0, 0, 0, 0, 0, 0, 0, 529, 530, 531, 489, 504, 485, 500, 448, 0, 0, 0, 0, 0, 0, 0, 0, 518, 451, 490, 528, 513, 523, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 524, 504, 489, 520, 496, 523, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4105, 4145, 4150, 4063, 4016, 4088, 4109] avg=4096.0000 max=4150 max_rank=3 min=4016 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 505, 501, 488, 469, 467, 530, 537, 13, 9, 1, 0, 0, 0, 0, 0, 534, 487, 541, 497, 513, 472, 521, 492, 31, 12, 0, 0, 0, 0, 0, 0, 468, 555, 478, 508, 517, 533, 508, 542, 0, 0, 0, 0, 0, 0, 0, 0, 523, 484, 498, 513, 514, 505, 527, 533, 0, 0, 0, 0, 0, 0, 0, 0, 546, 507, 490, 518, 522, 518, 488, 500, 4, 0, 0, 0, 0, 0, 0, 0, 473, 518, 525, 485, 511, 513, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 508, 530, 505, 501, 541, 537, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 505, 542, 526, 501, 526, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4100, 4109, 4097, 4093, 4040, 4174, 4126] avg=4096.0000 max=4174 max_rank=6 min=4029 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[539, 518, 526, 491, 539, 481, 488, 528, 13, 10, 1, 0, 0, 0, 0, 0, 487, 523, 486, 505, 492, 510, 482, 477, 32, 13, 0, 0, 0, 0, 0, 0, 527, 503, 513, 519, 515, 490, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 536, 546, 512, 503, 482, 528, 549, 477, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 498, 497, 495, 537, 516, 556, 4, 0, 0, 0, 0, 0, 0, 0, 503, 518, 523, 537, 471, 521, 537, 523, 0, 0, 0, 0, 0, 0, 0, 0, 472, 543, 494, 536, 514, 507, 487, 501, 0, 0, 0, 0, 0, 0, 0, 0, 548, 469, 476, 500, 485, 513, 535, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4007, 4109, 4133, 4127, 4133, 4054, 4071] avg=4096.0000 max=4134 max_rank=0 min=4007 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 540, 546, 497, 519, 520, 511, 482, 15, 9, 1, 0, 0, 0, 0, 0, 514, 514, 489, 511, 517, 479, 525, 510, 30, 14, 0, 0, 0, 0, 0, 0, 532, 470, 529, 488, 483, 486, 515, 498, 0, 0, 0, 0, 0, 0, 0, 0, 544, 535, 560, 514, 510, 542, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 521, 500, 490, 526, 488, 480, 490, 538, 4, 0, 0, 0, 0, 0, 0, 0, 505, 509, 532, 511, 532, 497, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 502, 516, 536, 484, 506, 462, 0, 0, 0, 0, 0, 0, 0, 0, 489, 544, 507, 499, 533, 510, 544, 473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4103, 4001, 4192, 4037, 4132, 4052, 4099] avg=4096.0000 max=4192 max_rank=3 min=4001 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 475, 516, 544, 516, 533, 504, 507, 12, 9, 1, 0, 0, 0, 0, 0, 513, 500, 485, 522, 508, 476, 507, 498, 32, 13, 0, 0, 0, 0, 0, 0, 530, 515, 512, 503, 503, 493, 517, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 513, 472, 530, 501, 492, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 540, 482, 503, 496, 533, 543, 498, 518, 4, 0, 0, 0, 0, 0, 0, 0, 501, 540, 466, 527, 511, 517, 522, 498, 0, 0, 0, 0, 0, 0, 0, 0, 477, 539, 491, 502, 500, 515, 544, 547, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 511, 487, 509, 506, 529, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4054, 4105, 4022, 4117, 4082, 4115, 4128] avg=4096.0000 max=4145 max_rank=0 min=4022 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[564, 535, 466, 488, 490, 500, 482, 520, 13, 9, 1, 0, 0, 0, 0, 0, 538, 496, 509, 524, 495, 503, 524, 510, 31, 13, 0, 0, 0, 0, 0, 0, 517, 512, 507, 507, 546, 507, 478, 530, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 508, 540, 492, 502, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 474, 517, 545, 526, 540, 500, 500, 501, 4, 0, 0, 0, 0, 0, 0, 0, 503, 533, 523, 504, 516, 509, 529, 465, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 536, 526, 514, 494, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 509, 538, 527, 517, 500, 478, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4143, 4104, 4092, 4107, 4082, 4102, 4070] avg=4096.0000 max=4143 max_rank=1 min=4068 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 490, 510, 539, 497, 507, 506, 486, 18, 14, 6, 2, 0, 0, 0, 0, 511, 504, 548, 496, 480, 484, 501, 545, 35, 18, 0, 0, 0, 0, 0, 0, 502, 535, 487, 504, 559, 528, 510, 516, 0, 0, 0, 0, 0, 0, 0, 0, 526, 499, 467, 510, 537, 517, 508, 480, 0, 0, 0, 0, 0, 0, 0, 0, 481, 531, 519, 509, 548, 522, 518, 513, 10, 0, 0, 0, 0, 0, 0, 0, 482, 506, 499, 546, 563, 486, 523, 483, 0, 0, 0, 0, 0, 0, 0, 0, 512, 525, 511, 538, 490, 489, 530, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 521, 517, 488, 466, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4122, 4141, 4044, 4151, 4088, 4095, 4048] avg=4096.0000 max=4151 max_rank=4 min=4044 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 513, 511, 484, 552, 473, 470, 527, 13, 9, 1, 0, 0, 0, 0, 0, 486, 539, 508, 501, 511, 547, 505, 492, 31, 14, 0, 0, 0, 0, 0, 0, 503, 496, 524, 512, 511, 518, 506, 484, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 509, 506, 497, 511, 494, 491, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 515, 483, 531, 473, 526, 497, 4, 0, 0, 0, 0, 0, 0, 0, 556, 514, 539, 497, 527, 529, 493, 540, 0, 0, 0, 0, 0, 0, 0, 0, 506, 486, 582, 492, 521, 500, 535, 466, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 506, 509, 530, 494, 518, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4134, 4054, 4038, 4073, 4195, 4088, 4117] avg=4096.0000 max=4195 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[520, 551, 508, 527, 512, 520, 484, 496, 496, 527, 490, 495, 491, 522, 477, 541, 498, 506, 528, 538, 481, 533, 500, 495, 509, 527, 497, 508, 550, 486, 537, 494, 537, 490, 513, 489, 516, 539, 519, 513, 540, 523, 478, 521, 474, 516, 541, 526, 543, 483, 508, 500, 493, 520, 524, 518, 527, 528, 483, 486, 549, 510, 496, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[524, 517, 527, 536, 489, 465, 471, 516, 505, 486, 481, 514, 509, 526, 493, 512, 487, 559, 531, 516, 500, 517, 553, 522, 522, 490, 556, 536, 497, 466, 523, 513, 498, 520, 473, 518, 529, 544, 491, 541, 529, 526, 550, 477, 559, 551, 485, 474, 507, 544, 499, 506, 506, 515, 475, 506, 509, 482, 544, 504, 497, 510, 541, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[543, 484, 517, 492, 541, 571, 502, 482, 514, 540, 505, 513, 524, 474, 495, 518, 493, 489, 509, 509, 487, 513, 544, 526, 466, 504, 496, 511, 471, 506, 523, 505, 525, 486, 504, 491, 493, 482, 514, 511, 536, 500, 510, 494, 499, 504, 540, 554, 540, 500, 534, 562, 491, 498, 549, 518, 545, 569, 510, 514, 515, 506, 493, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[520, 508, 544, 500, 529, 526, 485, 517, 482, 523, 492, 500, 507, 543, 487, 481, 463, 499, 499, 495, 487, 530, 534, 520, 520, 539, 508, 525, 486, 499, 506, 501, 530, 521, 518, 488, 529, 505, 541, 530, 537, 526, 524, 514, 526, 477, 495, 520, 497, 497, 526, 520, 530, 501, 544, 503, 494, 493, 505, 520, 549, 530, 485, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[518, 499, 506, 523, 528, 545, 500, 486, 500, 528, 506, 503, 492, 507, 495, 499, 509, 539, 510, 531, 525, 495, 540, 514, 524, 496, 498, 512, 509, 510, 506, 523, 539, 515, 483, 490, 557, 499, 534, 473, 500, 497, 492, 511, 522, 506, 505, 520, 520, 526, 537, 531, 495, 534, 509, 513, 534, 505, 528, 473, 504, 515, 484, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[506, 536, 494, 511, 522, 510, 526, 485, 512, 509, 520, 529, 541, 508, 504, 504, 525, 498, 474, 532, 485, 534, 511, 510, 492, 526, 510, 527, 501, 514, 503, 504, 530, 554, 488, 504, 488, 521, 522, 501, 520, 512, 507, 531, 476, 504, 518, 472, 530, 528, 517, 479, 509, 504, 551, 508, 536, 525, 464, 530, 527, 495, 541, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[520, 511, 544, 510, 552, 507, 506, 488, 538, 508, 518, 522, 503, 487, 483, 530, 502, 518, 517, 527, 463, 555, 512, 475, 449, 490, 504, 494, 488, 495, 507, 522, 512, 504, 538, 507, 520, 515, 543, 497, 510, 495, 540, 499, 484, 537, 476, 514, 530, 518, 524, 529, 528, 550, 532, 538, 499, 494, 538, 508, 485, 505, 517, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[542, 499, 513, 501, 493, 553, 488, 518, 483, 504, 534, 525, 508, 503, 495, 508, 503, 540, 549, 458, 501, 499, 557, 520, 546, 519, 522, 470, 483, 534, 518, 474, 535, 510, 531, 503, 520, 521, 506, 509, 529, 507, 512, 501, 560, 508, 534, 510, 510, 492, 503, 523, 502, 537, 520, 479, 488, 508, 494, 533, 524, 499, 490, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 551, 508, 527, 512, 509, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 490, 495, 491, 522, 477, 541, 11, 9, 8, 5, 2, 0, 0, 0, 498, 506, 528, 538, 481, 533, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 497, 508, 550, 486, 537, 494, 36, 3, 0, 0, 0, 0, 0, 0, 525, 490, 513, 489, 516, 539, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 532, 523, 478, 521, 474, 516, 541, 526, 0, 0, 0, 0, 0, 0, 0, 0, 543, 483, 508, 500, 493, 520, 488, 518, 0, 0, 0, 0, 0, 0, 0, 0, 527, 528, 483, 486, 549, 510, 496, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4074, 4077, 4147, 4104, 4111, 4053, 4095] avg=4096.0000 max=4147 max_rank=3 min=4053 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 517, 527, 536, 489, 455, 471, 516, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 481, 514, 509, 526, 493, 512, 10, 8, 8, 5, 2, 0, 0, 0, 487, 559, 531, 516, 500, 517, 551, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 556, 536, 497, 466, 523, 513, 33, 3, 0, 0, 0, 0, 0, 0, 487, 520, 473, 518, 529, 544, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 521, 526, 550, 477, 559, 551, 485, 474, 0, 0, 0, 0, 0, 0, 0, 0, 507, 544, 499, 506, 506, 515, 442, 506, 0, 0, 0, 0, 0, 0, 0, 0, 509, 482, 544, 504, 497, 510, 541, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4059, 4183, 4139, 4103, 4143, 4025, 4081] avg=4096.0000 max=4183 max_rank=2 min=4025 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 484, 517, 492, 541, 558, 502, 482, 0, 0, 0, 0, 0, 0, 0, 0, 514, 540, 505, 513, 524, 474, 495, 518, 13, 9, 8, 5, 2, 0, 0, 0, 493, 489, 509, 509, 487, 513, 542, 526, 0, 0, 0, 0, 0, 0, 0, 0, 466, 504, 496, 511, 471, 506, 523, 505, 38, 3, 0, 0, 0, 0, 0, 0, 513, 486, 504, 491, 493, 482, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 510, 494, 499, 504, 540, 554, 0, 0, 0, 0, 0, 0, 0, 0, 540, 500, 534, 562, 491, 498, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 545, 569, 510, 514, 515, 506, 493, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4120, 4068, 4023, 3994, 4129, 4154, 4161] avg=4096.0000 max=4161 max_rank=7 min=3994 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 508, 544, 500, 529, 514, 485, 517, 0, 0, 0, 0, 0, 0, 0, 0, 482, 523, 492, 500, 507, 543, 487, 481, 12, 9, 8, 5, 2, 0, 0, 0, 463, 499, 499, 495, 487, 530, 532, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 539, 508, 525, 486, 499, 506, 501, 38, 3, 0, 0, 0, 0, 0, 0, 518, 521, 518, 488, 529, 505, 541, 530, 0, 0, 0, 0, 0, 0, 0, 0, 529, 526, 524, 514, 526, 477, 495, 520, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 526, 520, 530, 501, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 494, 493, 505, 520, 549, 530, 485, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4051, 4025, 4125, 4150, 4111, 4080, 4109] avg=4096.0000 max=4150 max_rank=4 min=4025 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 499, 506, 523, 528, 533, 500, 486, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 506, 503, 492, 507, 495, 499, 12, 9, 7, 5, 2, 0, 0, 0, 509, 539, 510, 531, 525, 495, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 498, 512, 509, 510, 506, 523, 35, 3, 0, 0, 0, 0, 0, 0, 527, 515, 483, 490, 557, 499, 534, 473, 0, 0, 0, 0, 0, 0, 0, 0, 493, 497, 492, 511, 522, 506, 505, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 537, 531, 495, 534, 474, 513, 0, 0, 0, 0, 0, 0, 0, 0, 534, 505, 528, 473, 504, 515, 484, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4065, 4161, 4116, 4078, 4046, 4130, 4079] avg=4096.0000 max=4161 max_rank=2 min=4046 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[542, 499, 513, 501, 493, 541, 488, 518, 0, 0, 0, 0, 0, 0, 0, 0, 483, 504, 534, 525, 508, 503, 495, 508, 12, 9, 8, 5, 2, 0, 0, 0, 503, 540, 549, 458, 501, 499, 555, 520, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 522, 470, 483, 534, 518, 474, 36, 3, 0, 0, 0, 0, 0, 0, 523, 510, 531, 503, 520, 521, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 512, 501, 560, 508, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 510, 492, 503, 523, 502, 537, 484, 479, 0, 0, 0, 0, 0, 0, 0, 0, 488, 508, 494, 533, 524, 499, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4125, 4105, 4123, 4153, 4030, 4041] avg=4096.0000 max=4153 max_rank=5 min=4030 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 536, 494, 511, 522, 499, 526, 485, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 520, 529, 541, 508, 504, 504, 11, 9, 8, 5, 2, 0, 0, 0, 525, 498, 474, 532, 485, 534, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 510, 527, 501, 514, 503, 504, 38, 3, 0, 0, 0, 0, 0, 0, 518, 554, 488, 504, 488, 521, 522, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 507, 531, 476, 504, 518, 472, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 517, 479, 509, 504, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 536, 525, 464, 530, 527, 495, 541, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4162, 4067, 4118, 4096, 4032, 4088, 4126] avg=4096.0000 max=4162 max_rank=1 min=4032 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 511, 544, 510, 552, 492, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 518, 522, 503, 487, 483, 530, 15, 11, 12, 7, 7, 0, 0, 0, 502, 518, 517, 527, 463, 555, 505, 475, 0, 0, 0, 0, 0, 0, 0, 0, 449, 490, 504, 494, 488, 495, 507, 522, 41, 5, 0, 0, 0, 0, 0, 0, 496, 504, 538, 507, 520, 515, 543, 497, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 540, 499, 484, 537, 476, 514, 0, 0, 0, 0, 0, 0, 0, 0, 530, 518, 524, 529, 528, 550, 491, 538, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 538, 508, 485, 505, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4141, 4062, 3995, 4120, 4043, 4208, 4076] avg=4096.0000 max=4208 max_rank=6 min=3995 min_rank=3 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4352, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +Number of parameters in transformer block in billions: 20.11 [2026-08-26 15:43:11] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 37829.2 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097981E+01 | loss scale: 1.0 | grad norm: 29.763 | number of skipped iterations: 0 | number of nan iterations: 0 | + +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 68662.259765625 | max allocated: 75508.0107421875 | reserved: 80100.0 | max reserved: 80100.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[507, 509, 471, 524, 549, 555, 486, 514, 496, 530, 545, 513, 510, 480, 511, 514, 466, 545, 543, 496, 521, 486, 505, 502, 485, 467, 502, 517, 501, 493, 512, 498, 504, 536, 531, 518, 481, 538, 513, 511, 513, 508, 505, 525, 520, 481, 514, 508, 576, 536, 492, 503, 535, 538, 521, 491, 491, 463, 542, 523, 501, 585, 513, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[549, 494, 501, 490, 500, 494, 465, 514, 514, 488, 488, 539, 534, 523, 541, 506, 538, 557, 478, 506, 531, 502, 548, 516, 458, 530, 522, 520, 502, 528, 515, 527, 510, 486, 550, 499, 521, 525, 561, 527, 550, 525, 530, 493, 500, 498, 504, 532, 507, 514, 530, 500, 498, 497, 497, 502, 501, 484, 496, 525, 497, 513, 495, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[497, 536, 520, 502, 552, 506, 540, 497, 519, 496, 535, 495, 476, 486, 476, 520, 512, 479, 536, 491, 523, 499, 499, 494, 506, 486, 507, 536, 526, 541, 472, 512, 510, 478, 543, 502, 528, 536, 519, 507, 523, 481, 519, 534, 536, 488, 521, 513, 531, 518, 527, 528, 502, 509, 553, 511, 495, 549, 463, 515, 489, 545, 517, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[480, 501, 533, 535, 522, 498, 506, 485, 522, 508, 518, 505, 553, 516, 498, 524, 517, 512, 541, 516, 538, 500, 555, 469, 508, 504, 520, 506, 542, 508, 471, 538, 507, 497, 478, 511, 488, 539, 522, 517, 495, 522, 506, 509, 506, 557, 487, 486, 525, 512, 509, 539, 510, 528, 502, 491, 487, 527, 495, 511, 512, 524, 488, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[498, 484, 496, 517, 525, 532, 510, 492, 519, 526, 481, 521, 518, 506, 530, 478, 479, 517, 516, 528, 470, 522, 528, 521, 508, 463, 522, 505, 511, 534, 525, 514, 506, 482, 527, 489, 510, 481, 519, 515, 534, 503, 504, 508, 500, 493, 529, 525, 503, 511, 538, 525, 510, 532, 512, 514, 518, 489, 528, 515, 515, 533, 551, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[508, 534, 529, 491, 518, 506, 516, 575, 506, 520, 483, 533, 512, 527, 529, 498, 478, 552, 483, 510, 511, 541, 530, 509, 516, 467, 524, 523, 476, 508, 498, 505, 513, 535, 508, 494, 505, 532, 543, 510, 520, 500, 501, 497, 521, 502, 516, 513, 486, 511, 547, 529, 491, 482, 497, 523, 538, 493, 502, 482, 494, 533, 513, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[531, 555, 484, 487, 525, 532, 523, 519, 554, 479, 523, 510, 514, 516, 529, 520, 517, 475, 496, 514, 517, 523, 516, 488, 479, 527, 540, 540, 482, 535, 495, 502, 495, 504, 478, 518, 515, 529, 510, 526, 499, 521, 518, 492, 496, 530, 530, 559, 511, 517, 506, 560, 519, 527, 491, 490, 487, 483, 499, 463, 505, 538, 517, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[511, 531, 506, 508, 513, 511, 478, 502, 500, 541, 505, 518, 488, 513, 533, 510, 509, 522, 503, 510, 473, 492, 549, 518, 521, 536, 571, 506, 541, 505, 481, 497, 532, 505, 543, 534, 509, 492, 517, 504, 503, 543, 509, 484, 488, 494, 514, 504, 516, 535, 504, 497, 506, 512, 521, 556, 484, 501, 513, 507, 505, 498, 514, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 509, 471, 524, 548, 555, 486, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 530, 545, 513, 510, 480, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 466, 545, 543, 496, 521, 486, 505, 502, 3, 0, 0, 0, 0, 0, 0, 0, 485, 467, 502, 517, 501, 493, 512, 498, 12, 5, 0, 0, 0, 0, 0, 0, 504, 536, 531, 518, 481, 538, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 513, 508, 505, 525, 520, 481, 514, 508, 2, 1, 0, 0, 0, 0, 0, 0, 576, 536, 492, 474, 535, 538, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 491, 463, 542, 523, 501, 585, 513, 499, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4094, 4067, 3992, 4120, 4077, 4163, 4141] avg=4096.0000 max=4163 max_rank=6 min=3992 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[480, 501, 533, 535, 521, 498, 506, 485, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 518, 505, 553, 516, 493, 524, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 541, 516, 538, 500, 555, 469, 3, 0, 0, 0, 0, 0, 0, 0, 508, 504, 520, 506, 542, 508, 471, 538, 13, 5, 0, 0, 0, 0, 0, 0, 507, 497, 478, 511, 488, 539, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 506, 509, 506, 557, 487, 486, 2, 1, 0, 0, 0, 0, 0, 0, 525, 512, 509, 509, 510, 528, 502, 491, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 495, 511, 512, 524, 488, 532, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4139, 4151, 4115, 4046, 4071, 4086, 4101] avg=4096.0000 max=4151 max_rank=2 min=4046 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[549, 494, 501, 490, 496, 494, 465, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 488, 488, 539, 534, 523, 525, 506, 0, 0, 0, 0, 0, 0, 0, 0, 538, 557, 478, 506, 531, 502, 548, 516, 9, 0, 0, 0, 0, 0, 0, 0, 458, 530, 522, 520, 502, 528, 515, 527, 19, 8, 3, 0, 0, 0, 0, 0, 510, 486, 550, 499, 521, 525, 542, 527, 0, 0, 0, 0, 0, 0, 0, 0, 550, 525, 530, 493, 500, 498, 504, 532, 4, 4, 0, 0, 0, 0, 0, 0, 507, 514, 530, 465, 498, 497, 497, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 496, 525, 497, 513, 495, 483, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4117, 4185, 4132, 4160, 4140, 4010, 4021] avg=4096.0000 max=4185 max_rank=2 min=4003 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[497, 536, 520, 502, 551, 506, 540, 497, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 535, 495, 476, 486, 471, 520, 0, 0, 0, 0, 0, 0, 0, 0, 512, 479, 536, 491, 523, 499, 499, 494, 3, 0, 0, 0, 0, 0, 0, 0, 506, 486, 507, 536, 526, 541, 472, 512, 12, 5, 0, 0, 0, 0, 0, 0, 510, 478, 543, 502, 528, 536, 507, 507, 0, 0, 0, 0, 0, 0, 0, 0, 523, 481, 519, 534, 536, 488, 521, 513, 2, 1, 0, 0, 0, 0, 0, 0, 531, 518, 527, 498, 502, 509, 553, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 549, 463, 515, 489, 545, 517, 506, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3998, 4036, 4103, 4111, 4118, 4149, 4104] avg=4096.0000 max=4149 max_rank=0 min=3998 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 484, 496, 517, 524, 532, 510, 492, 0, 0, 0, 0, 0, 0, 0, 0, 519, 526, 481, 521, 518, 506, 525, 478, 0, 0, 0, 0, 0, 0, 0, 0, 479, 517, 516, 528, 470, 522, 528, 521, 3, 0, 0, 0, 0, 0, 0, 0, 508, 463, 522, 505, 511, 534, 525, 514, 12, 5, 0, 0, 0, 0, 0, 0, 506, 482, 527, 489, 510, 481, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 504, 508, 500, 493, 529, 525, 2, 1, 0, 0, 0, 0, 0, 0, 503, 511, 538, 495, 510, 532, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 528, 515, 515, 533, 551, 553, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4074, 4084, 4099, 4017, 4099, 4115, 4227] avg=4096.0000 max=4227 max_rank=7 min=4017 min_rank=4 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 529, 491, 517, 506, 516, 575, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 483, 533, 512, 527, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 478, 552, 483, 510, 511, 541, 530, 509, 3, 0, 0, 0, 0, 0, 0, 0, 516, 467, 524, 523, 476, 508, 498, 505, 13, 5, 0, 0, 0, 0, 0, 0, 513, 535, 508, 494, 505, 532, 530, 510, 0, 0, 0, 0, 0, 0, 0, 0, 520, 500, 501, 497, 521, 502, 516, 513, 2, 1, 0, 0, 0, 0, 0, 0, 486, 511, 547, 499, 491, 482, 497, 523, 0, 0, 0, 0, 0, 0, 0, 0, 538, 493, 502, 482, 494, 533, 513, 521, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4103, 4117, 4035, 4127, 4073, 4036, 4101] avg=4096.0000 max=4176 max_rank=0 min=4035 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[531, 555, 484, 487, 524, 532, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 554, 479, 523, 510, 514, 516, 524, 520, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 496, 514, 517, 523, 516, 488, 3, 0, 0, 0, 0, 0, 0, 0, 479, 527, 540, 540, 482, 535, 495, 502, 12, 5, 0, 0, 0, 0, 0, 0, 495, 504, 478, 518, 515, 529, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 518, 492, 496, 530, 530, 559, 2, 1, 0, 0, 0, 0, 0, 0, 511, 517, 506, 529, 519, 527, 491, 490, 0, 0, 0, 0, 0, 0, 0, 0, 487, 483, 499, 463, 505, 538, 517, 488, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4140, 4049, 4117, 4063, 4148, 4090, 4006] avg=4096.0000 max=4155 max_rank=0 min=4006 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[511, 531, 506, 508, 512, 511, 478, 502, 0, 0, 0, 0, 0, 0, 0, 0, 500, 541, 505, 518, 488, 513, 528, 510, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 503, 510, 473, 492, 549, 518, 3, 0, 0, 0, 0, 0, 0, 0, 521, 536, 571, 506, 541, 505, 481, 497, 12, 5, 0, 0, 0, 0, 0, 0, 532, 505, 543, 534, 509, 492, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 509, 484, 488, 494, 514, 504, 2, 1, 0, 0, 0, 0, 0, 0, 516, 535, 504, 469, 506, 512, 521, 556, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 513, 507, 505, 498, 514, 522, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4103, 4079, 4175, 4124, 4042, 4119, 4067] avg=4096.0000 max=4175 max_rank=3 min=4042 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4352, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[503, 563, 499, 530, 531, 493, 505, 488, 574, 514, 516, 461, 550, 518, 526, 494, 557, 492, 500, 514, 493, 544, 523, 498, 505, 530, 504, 514, 504, 488, 499, 481, 529, 519, 522, 529, 484, 512, 523, 527, 470, 508, 520, 538, 521, 514, 474, 487, 511, 501, 501, 508, 543, 504, 532, 457, 530, 532, 525, 512, 494, 497, 518, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[512, 539, 496, 487, 483, 526, 546, 535, 540, 508, 491, 549, 508, 549, 519, 537, 541, 507, 522, 521, 509, 505, 497, 519, 493, 516, 531, 518, 502, 487, 511, 485, 471, 539, 503, 502, 506, 532, 485, 491, 514, 537, 507, 489, 535, 483, 508, 524, 505, 515, 506, 505, 526, 496, 503, 492, 534, 498, 530, 502, 525, 501, 494, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[506, 523, 549, 514, 524, 483, 549, 505, 531, 494, 503, 502, 529, 524, 508, 486, 505, 511, 527, 536, 512, 539, 491, 464, 494, 522, 540, 512, 557, 481, 508, 473, 495, 496, 531, 494, 521, 508, 546, 504, 519, 508, 536, 519, 507, 508, 528, 477, 505, 523, 492, 534, 540, 484, 513, 483, 518, 507, 554, 507, 493, 524, 510, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[489, 522, 505, 481, 479, 503, 546, 503, 509, 478, 489, 510, 530, 571, 512, 532, 500, 494, 512, 499, 516, 519, 515, 515, 503, 525, 479, 489, 536, 497, 518, 520, 496, 484, 517, 532, 497, 514, 516, 516, 514, 486, 527, 539, 490, 525, 544, 531, 470, 513, 560, 538, 493, 511, 485, 520, 532, 525, 491, 521, 485, 548, 519, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[538, 510, 530, 499, 572, 511, 537, 515, 462, 506, 529, 512, 563, 488, 522, 492, 528, 502, 519, 517, 510, 545, 484, 470, 529, 520, 521, 519, 535, 480, 476, 510, 482, 483, 538, 509, 493, 506, 517, 552, 504, 490, 497, 534, 501, 533, 529, 500, 530, 505, 535, 525, 509, 506, 501, 566, 512, 532, 476, 495, 510, 466, 496, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[490, 518, 498, 531, 509, 481, 493, 519, 521, 535, 505, 564, 506, 481, 462, 499, 512, 503, 485, 499, 520, 521, 504, 529, 525, 527, 490, 506, 536, 522, 509, 521, 504, 508, 508, 520, 491, 529, 507, 524, 529, 480, 504, 532, 513, 522, 541, 513, 535, 514, 515, 546, 493, 514, 526, 519, 500, 493, 492, 515, 521, 506, 510, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[516, 508, 537, 505, 501, 476, 489, 520, 487, 514, 517, 536, 542, 526, 482, 546, 519, 544, 527, 531, 524, 511, 515, 530, 472, 520, 498, 498, 486, 456, 550, 527, 500, 523, 506, 502, 510, 523, 517, 493, 515, 506, 509, 536, 503, 509, 545, 523, 494, 491, 483, 523, 484, 492, 516, 511, 511, 495, 546, 540, 475, 518, 537, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[485, 537, 450, 544, 492, 514, 516, 495, 474, 488, 501, 508, 543, 512, 508, 510, 493, 505, 506, 486, 508, 556, 481, 502, 482, 506, 511, 494, 530, 540, 517, 534, 525, 513, 572, 517, 544, 470, 493, 496, 539, 533, 540, 523, 507, 525, 517, 531, 558, 520, 501, 466, 514, 502, 512, 503, 531, 497, 564, 466, 482, 502, 549, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[512, 539, 496, 487, 483, 526, 546, 535, 1, 0, 0, 0, 0, 0, 0, 0, 540, 508, 491, 549, 493, 549, 519, 537, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 522, 521, 509, 504, 497, 519, 0, 0, 0, 0, 0, 0, 0, 0, 493, 516, 531, 518, 502, 487, 511, 485, 23, 0, 0, 0, 0, 0, 0, 0, 471, 539, 503, 502, 506, 532, 485, 491, 5, 0, 0, 0, 0, 0, 0, 0, 514, 537, 507, 466, 535, 483, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 506, 505, 526, 496, 503, 492, 10, 0, 0, 0, 0, 0, 0, 0, 534, 498, 530, 502, 525, 501, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4186, 4120, 4066, 4034, 4074, 4058, 4105] avg=4096.0000 max=4186 max_rank=1 min=4034 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 522, 505, 481, 479, 503, 546, 503, 1, 0, 0, 0, 0, 0, 0, 0, 509, 478, 489, 510, 515, 571, 512, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 512, 499, 516, 518, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 525, 479, 489, 536, 497, 518, 520, 25, 0, 0, 0, 0, 0, 0, 0, 496, 484, 517, 532, 497, 514, 516, 516, 5, 0, 0, 0, 0, 0, 0, 0, 514, 486, 527, 514, 490, 525, 544, 531, 0, 0, 0, 0, 0, 0, 0, 0, 470, 513, 560, 538, 493, 511, 485, 520, 10, 0, 0, 0, 0, 0, 0, 0, 532, 525, 491, 521, 485, 548, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4116, 4069, 4092, 4077, 4131, 4100, 4154] avg=4096.0000 max=4154 max_rank=7 min=4029 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[503, 563, 499, 530, 531, 493, 505, 488, 4, 4, 0, 0, 0, 0, 0, 0, 574, 514, 516, 461, 526, 518, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 557, 492, 500, 514, 493, 536, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 504, 514, 504, 488, 499, 481, 28, 0, 0, 0, 0, 0, 0, 0, 529, 519, 522, 529, 484, 512, 523, 527, 10, 4, 0, 0, 0, 0, 0, 0, 470, 508, 520, 507, 521, 514, 474, 487, 0, 0, 0, 0, 0, 0, 0, 0, 511, 501, 501, 508, 543, 504, 532, 457, 14, 3, 0, 0, 0, 0, 0, 0, 530, 532, 521, 512, 494, 497, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4129, 4113, 4053, 4159, 4001, 4074, 4119] avg=4096.0000 max=4159 max_rank=4 min=4001 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 549, 514, 524, 483, 549, 505, 1, 0, 0, 0, 0, 0, 0, 0, 531, 494, 503, 502, 514, 524, 508, 486, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 527, 536, 512, 538, 491, 464, 0, 0, 0, 0, 0, 0, 0, 0, 494, 522, 540, 512, 557, 481, 508, 473, 24, 0, 0, 0, 0, 0, 0, 0, 495, 496, 531, 494, 521, 508, 546, 504, 5, 0, 0, 0, 0, 0, 0, 0, 519, 508, 536, 495, 507, 508, 528, 477, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 492, 534, 540, 484, 513, 483, 10, 0, 0, 0, 0, 0, 0, 0, 518, 507, 554, 507, 493, 524, 510, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4062, 4084, 4111, 4100, 4078, 4084, 4095] avg=4096.0000 max=4154 max_rank=0 min=4062 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[538, 510, 530, 499, 572, 511, 537, 515, 1, 0, 0, 0, 0, 0, 0, 0, 462, 506, 529, 512, 546, 488, 522, 492, 0, 0, 0, 0, 0, 0, 0, 0, 528, 502, 519, 517, 510, 544, 484, 470, 0, 0, 0, 0, 0, 0, 0, 0, 529, 520, 521, 519, 535, 480, 476, 510, 25, 0, 0, 0, 0, 0, 0, 0, 482, 483, 538, 509, 493, 506, 517, 552, 6, 0, 0, 0, 0, 0, 0, 0, 504, 490, 497, 509, 501, 533, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 535, 525, 509, 506, 501, 566, 11, 0, 0, 0, 0, 0, 0, 0, 512, 532, 476, 495, 510, 466, 496, 485, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4057, 4074, 4115, 4086, 4063, 4188, 3972] avg=4096.0000 max=4213 max_rank=0 min=3972 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 518, 498, 531, 509, 481, 493, 519, 1, 0, 0, 0, 0, 0, 0, 0, 521, 535, 505, 564, 491, 481, 462, 499, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 485, 499, 520, 520, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 490, 506, 536, 522, 509, 521, 25, 0, 0, 0, 0, 0, 0, 0, 504, 508, 508, 520, 491, 529, 507, 524, 5, 0, 0, 0, 0, 0, 0, 0, 529, 480, 504, 507, 513, 522, 541, 513, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 515, 546, 493, 514, 526, 519, 10, 0, 0, 0, 0, 0, 0, 0, 500, 493, 492, 515, 521, 506, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4058, 4072, 4161, 4096, 4109, 4172, 4060] avg=4096.0000 max=4172 max_rank=6 min=4040 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[485, 537, 450, 544, 492, 514, 516, 495, 1, 0, 0, 0, 0, 0, 0, 0, 474, 488, 501, 508, 526, 512, 508, 510, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 506, 486, 508, 555, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 482, 506, 511, 494, 530, 540, 517, 534, 24, 0, 0, 0, 0, 0, 0, 0, 525, 513, 572, 517, 544, 470, 493, 496, 6, 0, 0, 0, 0, 0, 0, 0, 539, 533, 540, 499, 507, 525, 517, 531, 0, 0, 0, 0, 0, 0, 0, 0, 558, 520, 501, 466, 514, 502, 512, 503, 11, 0, 0, 0, 0, 0, 0, 0, 531, 497, 564, 466, 482, 502, 549, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4027, 4036, 4138, 4136, 4191, 4087, 4119] avg=4096.0000 max=4191 max_rank=5 min=4027 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[516, 508, 537, 505, 501, 476, 489, 520, 1, 0, 0, 0, 0, 0, 0, 0, 487, 514, 517, 536, 525, 526, 482, 546, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 527, 531, 524, 510, 515, 530, 0, 0, 0, 0, 0, 0, 0, 0, 472, 520, 498, 498, 486, 456, 550, 527, 25, 0, 0, 0, 0, 0, 0, 0, 500, 523, 506, 502, 510, 523, 517, 493, 6, 0, 0, 0, 0, 0, 0, 0, 515, 506, 509, 511, 503, 509, 545, 523, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 483, 523, 484, 492, 516, 511, 11, 0, 0, 0, 0, 0, 0, 0, 511, 495, 546, 540, 475, 518, 537, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4133, 4200, 4032, 4080, 4121, 4005, 4144] avg=4096.0000 max=4200 max_rank=2 min=4005 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4352, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[500, 537, 508, 579, 520, 452, 522, 488, 511, 494, 532, 537, 510, 522, 512, 504, 547, 510, 493, 523, 502, 538, 531, 502, 514, 506, 468, 530, 536, 496, 514, 519, 499, 509, 512, 480, 528, 544, 526, 502, 496, 522, 514, 520, 506, 502, 523, 497, 511, 502, 503, 531, 495, 529, 521, 510, 522, 480, 510, 485, 523, 509, 488, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[503, 510, 476, 489, 526, 489, 513, 500, 515, 525, 520, 494, 502, 539, 502, 505, 505, 458, 492, 528, 515, 518, 522, 511, 535, 538, 530, 519, 511, 477, 523, 499, 600, 511, 514, 523, 480, 502, 486, 549, 539, 498, 515, 521, 529, 508, 494, 524, 490, 494, 520, 534, 480, 526, 496, 479, 499, 562, 531, 486, 548, 519, 520, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[537, 497, 485, 573, 543, 538, 519, 506, 541, 539, 521, 511, 529, 549, 505, 501, 522, 471, 526, 549, 507, 519, 550, 551, 501, 493, 525, 459, 542, 483, 504, 499, 526, 497, 512, 524, 494, 485, 514, 474, 522, 519, 513, 462, 490, 531, 483, 494, 491, 498, 541, 480, 498, 516, 515, 505, 468, 538, 530, 509, 514, 494, 532, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[490, 484, 483, 529, 516, 514, 519, 524, 486, 481, 516, 542, 523, 513, 488, 504, 503, 485, 475, 550, 526, 521, 515, 509, 490, 538, 501, 464, 523, 527, 515, 522, 482, 532, 529, 517, 540, 514, 508, 491, 523, 523, 515, 516, 527, 498, 534, 500, 535, 520, 522, 511, 513, 517, 528, 527, 507, 508, 516, 537, 483, 505, 505, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[480, 469, 502, 493, 515, 505, 485, 483, 482, 518, 532, 521, 549, 470, 513, 515, 512, 528, 493, 521, 513, 503, 498, 531, 505, 520, 521, 520, 522, 529, 561, 459, 518, 529, 520, 493, 516, 531, 479, 512, 508, 524, 531, 493, 556, 516, 540, 523, 510, 516, 526, 487, 525, 502, 502, 496, 485, 506, 538, 521, 507, 530, 543, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[503, 532, 531, 531, 502, 514, 527, 518, 519, 518, 531, 517, 502, 542, 494, 493, 502, 501, 524, 517, 541, 505, 509, 500, 465, 475, 535, 500, 517, 519, 540, 507, 520, 518, 542, 474, 518, 531, 504, 526, 504, 530, 475, 535, 516, 497, 491, 465, 515, 520, 496, 573, 512, 496, 522, 524, 501, 502, 476, 527, 537, 487, 521, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[528, 533, 512, 518, 512, 491, 494, 479, 503, 544, 502, 553, 522, 523, 479, 538, 520, 520, 504, 520, 508, 492, 527, 497, 520, 508, 513, 527, 527, 499, 557, 519, 464, 506, 477, 529, 524, 522, 535, 510, 516, 510, 526, 524, 498, 541, 521, 489, 482, 532, 496, 484, 532, 532, 523, 462, 513, 468, 518, 524, 494, 516, 500, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[489, 499, 532, 478, 515, 496, 533, 509, 517, 528, 531, 488, 515, 506, 540, 527, 506, 517, 504, 522, 494, 475, 520, 514, 520, 518, 516, 528, 507, 490, 482, 520, 531, 512, 503, 484, 528, 531, 482, 500, 520, 516, 522, 534, 542, 510, 501, 506, 518, 511, 542, 501, 537, 479, 489, 520, 499, 510, 527, 548, 487, 496, 526, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 537, 508, 579, 520, 452, 522, 488, 24, 0, 0, 0, 0, 0, 0, 0, 511, 494, 503, 537, 510, 522, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 547, 510, 493, 519, 502, 538, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 468, 530, 536, 496, 514, 519, 1, 0, 0, 0, 0, 0, 0, 0, 499, 509, 512, 480, 528, 544, 526, 502, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 514, 520, 499, 502, 523, 497, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 503, 531, 495, 529, 521, 510, 4, 3, 0, 0, 0, 0, 0, 0, 522, 480, 510, 485, 523, 509, 488, 512, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4093, 4142, 4084, 4100, 4073, 4109, 4037] avg=4096.0000 max=4142 max_rank=2 min=4037 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[503, 510, 476, 489, 526, 489, 513, 500, 23, 0, 0, 0, 0, 0, 0, 0, 515, 525, 492, 494, 502, 539, 502, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 458, 492, 524, 515, 518, 522, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 538, 530, 519, 511, 477, 523, 499, 1, 0, 0, 0, 0, 0, 0, 0, 600, 511, 514, 523, 480, 502, 486, 549, 0, 0, 0, 0, 0, 0, 0, 0, 539, 498, 515, 521, 521, 508, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 490, 494, 520, 534, 480, 526, 496, 479, 5, 3, 0, 0, 0, 0, 0, 0, 499, 562, 531, 486, 548, 519, 520, 502, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4074, 4045, 4133, 4165, 4120, 4027, 4175] avg=4096.0000 max=4175 max_rank=7 min=4027 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[480, 469, 502, 493, 515, 505, 485, 483, 24, 0, 0, 0, 0, 0, 0, 0, 482, 518, 503, 521, 549, 470, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 493, 517, 513, 503, 498, 531, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 521, 520, 522, 529, 561, 459, 1, 0, 0, 0, 0, 0, 0, 0, 518, 529, 520, 493, 516, 531, 479, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 531, 493, 547, 516, 540, 523, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 526, 487, 525, 502, 502, 496, 5, 3, 0, 0, 0, 0, 0, 0, 485, 506, 538, 521, 507, 530, 543, 517, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 4071, 4095, 4138, 4098, 4182, 4072, 4156] avg=4096.0000 max=4182 max_rank=5 min=3956 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 485, 573, 543, 538, 519, 506, 23, 0, 0, 0, 0, 0, 0, 0, 541, 539, 493, 511, 529, 549, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 522, 471, 526, 544, 507, 519, 550, 551, 0, 0, 0, 0, 0, 0, 0, 0, 501, 493, 525, 459, 542, 483, 504, 499, 2, 0, 0, 0, 0, 0, 0, 0, 526, 497, 512, 524, 494, 485, 514, 474, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 513, 462, 483, 531, 483, 494, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 541, 480, 498, 516, 515, 505, 4, 3, 0, 0, 0, 0, 0, 0, 468, 538, 530, 509, 514, 494, 532, 504, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4168, 4190, 4008, 4026, 4007, 4051, 4097] avg=4096.0000 max=4221 max_rank=0 min=4007 min_rank=5 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16181 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[490, 484, 483, 529, 516, 514, 519, 524, 23, 0, 0, 0, 0, 0, 0, 0, 486, 481, 488, 542, 523, 513, 488, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 485, 475, 545, 526, 521, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 490, 538, 501, 464, 523, 527, 515, 522, 2, 0, 0, 0, 0, 0, 0, 0, 482, 532, 529, 517, 540, 514, 508, 491, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 515, 516, 519, 498, 534, 500, 0, 0, 0, 0, 0, 0, 0, 0, 535, 520, 522, 511, 513, 517, 528, 527, 5, 3, 0, 0, 0, 0, 0, 0, 507, 508, 516, 537, 483, 505, 505, 509, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4025, 4079, 4082, 4113, 4128, 4181, 4078] avg=4096.0000 max=4181 max_rank=6 min=4025 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[503, 532, 531, 531, 502, 514, 527, 518, 28, 0, 0, 0, 0, 0, 0, 0, 519, 518, 492, 517, 502, 542, 494, 493, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 524, 503, 541, 505, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 465, 475, 535, 500, 517, 519, 540, 507, 7, 5, 0, 0, 0, 0, 0, 0, 520, 518, 542, 474, 518, 526, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 504, 530, 475, 535, 500, 497, 491, 465, 0, 0, 0, 0, 0, 0, 0, 0, 515, 520, 496, 573, 512, 496, 522, 524, 8, 7, 0, 0, 0, 0, 0, 0, 501, 502, 476, 527, 537, 487, 521, 482, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4077, 4085, 4070, 4128, 3997, 4173, 4052] avg=4096.0000 max=4186 max_rank=0 min=3997 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[528, 533, 512, 518, 512, 491, 494, 479, 22, 0, 0, 0, 0, 0, 0, 0, 503, 544, 475, 553, 522, 523, 479, 538, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 504, 516, 508, 492, 527, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 513, 527, 527, 499, 557, 519, 1, 0, 0, 0, 0, 0, 0, 0, 464, 506, 477, 529, 524, 522, 535, 510, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 526, 524, 491, 541, 521, 489, 0, 0, 0, 0, 0, 0, 0, 0, 482, 532, 496, 484, 532, 532, 523, 462, 4, 3, 0, 0, 0, 0, 0, 0, 513, 468, 518, 524, 494, 516, 500, 511, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4137, 4084, 4171, 4067, 4118, 4050, 4052] avg=4096.0000 max=4171 max_rank=3 min=4050 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[489, 499, 532, 478, 515, 496, 533, 509, 24, 0, 0, 0, 0, 0, 0, 0, 517, 528, 502, 488, 515, 506, 540, 527, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 504, 518, 494, 475, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 516, 528, 507, 490, 482, 520, 1, 0, 0, 0, 0, 0, 0, 0, 531, 512, 503, 484, 528, 531, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 522, 534, 534, 510, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 542, 501, 537, 479, 489, 520, 5, 3, 0, 0, 0, 0, 0, 0, 499, 510, 527, 548, 487, 496, 526, 520, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4123, 4048, 4082, 4071, 4143, 4105, 4121] avg=4096.0000 max=4143 max_rank=5 min=4048 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[486, 514, 545, 521, 511, 488, 519, 497, 549, 493, 498, 519, 498, 545, 538, 576, 525, 557, 499, 520, 501, 490, 533, 499, 491, 478, 486, 491, 514, 540, 511, 514, 515, 526, 495, 536, 514, 537, 509, 494, 525, 513, 517, 483, 501, 487, 528, 518, 499, 539, 505, 551, 487, 531, 458, 486, 506, 523, 513, 517, 519, 488, 495, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[509, 523, 535, 523, 514, 519, 514, 533, 497, 523, 522, 473, 473, 522, 524, 520, 481, 504, 514, 498, 497, 453, 493, 522, 449, 509, 478, 532, 498, 545, 526, 505, 528, 553, 510, 524, 536, 498, 482, 523, 523, 517, 519, 515, 496, 522, 572, 526, 533, 495, 483, 535, 505, 564, 512, 483, 510, 514, 549, 495, 500, 546, 470, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[527, 514, 528, 513, 504, 555, 517, 480, 524, 522, 511, 500, 512, 519, 514, 547, 535, 484, 526, 486, 494, 504, 494, 511, 522, 513, 530, 487, 502, 498, 503, 542, 564, 499, 488, 533, 510, 498, 494, 488, 508, 544, 537, 489, 519, 503, 511, 532, 511, 484, 471, 510, 508, 515, 496, 508, 516, 529, 484, 526, 515, 516, 560, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[577, 532, 503, 507, 524, 532, 522, 495, 518, 501, 473, 508, 473, 539, 533, 464, 529, 512, 542, 491, 506, 522, 524, 531, 531, 537, 553, 502, 514, 507, 509, 511, 542, 508, 514, 513, 501, 507, 531, 511, 512, 508, 507, 511, 480, 516, 515, 495, 523, 482, 532, 514, 506, 472, 500, 536, 501, 542, 504, 484, 523, 479, 467, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[501, 494, 532, 516, 534, 518, 553, 485, 547, 527, 499, 523, 558, 502, 497, 517, 535, 517, 515, 514, 579, 521, 492, 497, 513, 490, 502, 552, 507, 506, 492, 524, 516, 530, 484, 484, 496, 475, 530, 494, 482, 517, 502, 508, 519, 508, 494, 486, 529, 507, 488, 505, 474, 521, 501, 532, 509, 526, 519, 521, 500, 523, 490, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[523, 529, 507, 557, 528, 508, 492, 529, 483, 544, 523, 508, 493, 500, 513, 509, 511, 504, 494, 510, 501, 479, 505, 491, 483, 523, 562, 539, 524, 487, 527, 511, 493, 521, 505, 527, 554, 470, 547, 502, 517, 530, 491, 532, 532, 495, 504, 497, 508, 549, 538, 500, 541, 495, 506, 522, 486, 525, 463, 467, 516, 535, 510, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[486, 534, 536, 487, 535, 535, 512, 468, 484, 512, 552, 519, 497, 512, 517, 509, 526, 528, 530, 496, 459, 506, 515, 516, 501, 531, 511, 518, 494, 549, 508, 495, 543, 512, 493, 499, 548, 512, 508, 495, 521, 505, 517, 517, 477, 518, 515, 491, 553, 491, 510, 514, 492, 536, 521, 493, 497, 525, 517, 555, 548, 484, 478, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[523, 504, 500, 535, 538, 495, 502, 510, 516, 517, 480, 482, 537, 500, 511, 511, 486, 507, 539, 507, 522, 562, 512, 562, 507, 470, 505, 486, 483, 523, 484, 528, 541, 537, 476, 524, 530, 482, 516, 525, 489, 561, 524, 506, 508, 513, 513, 517, 495, 503, 516, 526, 534, 526, 480, 495, 521, 520, 483, 531, 515, 516, 505, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 514, 528, 513, 467, 555, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 511, 500, 512, 519, 514, 540, 0, 0, 0, 0, 0, 0, 0, 0, 535, 484, 526, 486, 494, 504, 494, 511, 6, 0, 0, 0, 0, 0, 0, 0, 522, 513, 530, 487, 502, 498, 503, 542, 9, 3, 0, 0, 0, 0, 0, 0, 555, 499, 488, 533, 510, 498, 494, 488, 0, 0, 0, 0, 0, 0, 0, 0, 508, 544, 537, 489, 519, 503, 511, 532, 1, 0, 0, 0, 0, 0, 0, 0, 511, 484, 471, 510, 508, 515, 496, 508, 16, 0, 0, 0, 0, 0, 0, 0, 516, 529, 484, 526, 515, 516, 560, 484, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4142, 4040, 4109, 4065, 4144, 4019, 4148] avg=4096.0000 max=4148 max_rank=7 min=4019 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[486, 514, 545, 521, 473, 488, 519, 497, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 498, 519, 498, 545, 538, 569, 0, 0, 0, 0, 0, 0, 0, 0, 525, 557, 499, 520, 501, 490, 533, 499, 6, 0, 0, 0, 0, 0, 0, 0, 491, 478, 486, 491, 514, 540, 511, 514, 9, 3, 0, 0, 0, 0, 0, 0, 506, 526, 495, 536, 514, 537, 509, 494, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 517, 483, 501, 487, 528, 518, 1, 0, 0, 0, 0, 0, 0, 0, 499, 539, 505, 551, 487, 531, 458, 486, 16, 0, 0, 0, 0, 0, 0, 0, 506, 523, 513, 517, 519, 488, 495, 507, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4209, 4130, 4037, 4117, 4073, 4072, 4087] avg=4096.0000 max=4209 max_rank=1 min=4037 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[577, 532, 503, 507, 486, 532, 522, 495, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 473, 508, 473, 539, 533, 458, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 542, 491, 506, 522, 524, 531, 5, 0, 0, 0, 0, 0, 0, 0, 531, 537, 553, 502, 514, 507, 509, 511, 9, 3, 0, 0, 0, 0, 0, 0, 533, 508, 514, 513, 501, 507, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 507, 511, 480, 516, 515, 495, 1, 0, 0, 0, 0, 0, 0, 0, 523, 482, 532, 514, 506, 472, 500, 536, 16, 0, 0, 0, 0, 0, 0, 0, 501, 542, 504, 484, 523, 479, 467, 510, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4003, 4162, 4176, 4118, 4045, 4081, 4029] avg=4096.0000 max=4176 max_rank=3 min=4003 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 523, 535, 523, 462, 519, 514, 533, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 522, 473, 473, 522, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 514, 498, 497, 453, 493, 522, 9, 7, 0, 0, 0, 0, 0, 0, 449, 509, 478, 532, 498, 545, 526, 505, 13, 9, 0, 0, 0, 0, 0, 0, 508, 553, 510, 524, 536, 498, 482, 523, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 519, 515, 496, 522, 572, 526, 6, 0, 0, 0, 0, 0, 0, 0, 533, 495, 483, 535, 505, 564, 512, 483, 20, 0, 0, 0, 0, 0, 0, 0, 510, 514, 549, 495, 500, 546, 470, 502, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4039, 3978, 4064, 4134, 4196, 4130, 4109] avg=4096.0000 max=4196 max_rank=5 min=3978 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[486, 534, 536, 487, 497, 535, 512, 468, 0, 0, 0, 0, 0, 0, 0, 0, 484, 512, 552, 519, 497, 512, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 530, 496, 459, 506, 515, 516, 5, 0, 0, 0, 0, 0, 0, 0, 501, 531, 511, 518, 494, 549, 508, 495, 9, 3, 0, 0, 0, 0, 0, 0, 534, 512, 493, 499, 548, 512, 508, 495, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 517, 517, 477, 518, 515, 491, 1, 0, 0, 0, 0, 0, 0, 0, 553, 491, 510, 514, 492, 536, 521, 493, 16, 0, 0, 0, 0, 0, 0, 0, 497, 525, 517, 555, 548, 484, 478, 505, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4096, 4081, 4119, 4101, 4062, 4126, 4128] avg=4096.0000 max=4128 max_rank=7 min=4055 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[523, 529, 507, 557, 490, 508, 492, 529, 0, 0, 0, 0, 0, 0, 0, 0, 483, 544, 523, 508, 493, 500, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 494, 510, 501, 479, 505, 491, 5, 0, 0, 0, 0, 0, 0, 0, 483, 523, 562, 539, 524, 487, 527, 511, 8, 3, 0, 0, 0, 0, 0, 0, 485, 521, 505, 527, 554, 470, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 517, 530, 491, 532, 532, 495, 504, 497, 1, 0, 0, 0, 0, 0, 0, 0, 508, 549, 538, 500, 541, 495, 506, 522, 16, 0, 0, 0, 0, 0, 0, 0, 486, 525, 463, 467, 516, 535, 510, 493, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4067, 4000, 4167, 4111, 4099, 4175, 4014] avg=4096.0000 max=4175 max_rank=6 min=4000 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 494, 532, 516, 496, 518, 553, 485, 0, 0, 0, 0, 0, 0, 0, 0, 547, 527, 499, 523, 558, 502, 497, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 515, 514, 579, 521, 492, 497, 5, 0, 0, 0, 0, 0, 0, 0, 513, 490, 502, 552, 507, 506, 492, 524, 9, 3, 0, 0, 0, 0, 0, 0, 507, 530, 484, 484, 496, 475, 530, 494, 0, 0, 0, 0, 0, 0, 0, 0, 482, 517, 502, 508, 519, 508, 494, 486, 1, 0, 0, 0, 0, 0, 0, 0, 529, 507, 488, 505, 474, 521, 501, 532, 16, 0, 0, 0, 0, 0, 0, 0, 509, 526, 519, 521, 500, 523, 490, 539, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4164, 4175, 4098, 4000, 4017, 4073, 4146] avg=4096.0000 max=4175 max_rank=2 min=4000 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16236 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[523, 504, 500, 535, 498, 495, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 480, 482, 537, 500, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 539, 507, 522, 562, 512, 562, 5, 0, 0, 0, 0, 0, 0, 0, 507, 470, 505, 486, 483, 523, 484, 528, 9, 3, 0, 0, 0, 0, 0, 0, 532, 537, 476, 524, 530, 482, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 489, 561, 524, 506, 508, 513, 513, 517, 1, 0, 0, 0, 0, 0, 0, 0, 495, 503, 516, 526, 534, 526, 480, 495, 17, 0, 0, 0, 0, 0, 0, 0, 521, 520, 483, 531, 515, 516, 505, 496, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4048, 4202, 3998, 4122, 4132, 4092, 4107] avg=4096.0000 max=4202 max_rank=2 min=3998 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[502, 534, 506, 514, 536, 506, 497, 506, 496, 492, 499, 500, 542, 518, 509, 508, 531, 570, 512, 512, 492, 541, 505, 529, 485, 448, 544, 537, 536, 510, 522, 515, 503, 507, 498, 508, 523, 470, 481, 520, 531, 508, 507, 542, 492, 498, 538, 517, 507, 513, 517, 526, 496, 518, 501, 507, 511, 533, 523, 520, 502, 505, 521, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[517, 537, 486, 518, 526, 499, 495, 485, 543, 509, 503, 499, 487, 490, 525, 537, 501, 532, 548, 486, 527, 508, 499, 534, 514, 512, 514, 469, 528, 486, 522, 509, 509, 518, 508, 495, 525, 509, 512, 521, 510, 490, 507, 542, 486, 526, 512, 495, 485, 494, 493, 516, 525, 510, 549, 520, 472, 508, 530, 527, 522, 527, 537, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[557, 467, 513, 516, 551, 506, 529, 499, 500, 541, 517, 534, 524, 533, 458, 494, 494, 548, 519, 511, 499, 535, 512, 491, 493, 542, 500, 501, 523, 509, 470, 526, 487, 549, 560, 519, 536, 479, 502, 510, 534, 524, 510, 501, 470, 493, 512, 527, 494, 543, 469, 508, 513, 510, 505, 526, 529, 513, 507, 465, 535, 515, 507, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[471, 516, 502, 528, 498, 488, 505, 502, 509, 539, 547, 546, 500, 539, 522, 523, 498, 475, 510, 521, 530, 494, 516, 527, 488, 507, 477, 519, 510, 516, 516, 512, 517, 533, 534, 522, 496, 524, 459, 538, 509, 506, 495, 531, 511, 527, 451, 520, 550, 509, 496, 508, 490, 517, 564, 478, 556, 546, 478, 497, 521, 535, 505, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[511, 515, 504, 500, 509, 513, 515, 535, 494, 527, 509, 508, 517, 503, 545, 516, 541, 488, 501, 488, 477, 524, 508, 528, 546, 565, 522, 543, 502, 491, 491, 518, 520, 532, 505, 509, 484, 494, 506, 514, 507, 487, 500, 484, 490, 561, 524, 524, 468, 503, 508, 537, 518, 503, 504, 469, 557, 503, 535, 541, 510, 524, 501, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[511, 503, 480, 556, 509, 535, 531, 513, 516, 473, 509, 467, 495, 536, 516, 515, 487, 523, 545, 551, 536, 544, 508, 509, 522, 514, 477, 514, 482, 500, 536, 508, 505, 521, 509, 504, 469, 524, 500, 516, 482, 505, 545, 497, 514, 532, 533, 463, 535, 515, 511, 505, 530, 509, 513, 468, 502, 513, 463, 541, 508, 543, 565, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[486, 510, 512, 514, 494, 512, 540, 508, 465, 519, 494, 480, 487, 522, 484, 540, 515, 504, 528, 530, 489, 525, 518, 503, 522, 536, 493, 536, 514, 564, 495, 499, 488, 546, 493, 542, 518, 494, 508, 515, 510, 531, 527, 497, 505, 489, 536, 497, 529, 511, 507, 507, 529, 493, 494, 527, 494, 488, 521, 517, 496, 517, 578, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[567, 525, 496, 543, 504, 487, 498, 491, 504, 547, 524, 526, 533, 523, 527, 486, 500, 504, 519, 526, 532, 486, 502, 496, 505, 513, 497, 488, 511, 513, 512, 502, 509, 519, 531, 519, 515, 527, 491, 510, 503, 512, 503, 517, 505, 509, 525, 513, 518, 501, 513, 520, 462, 507, 530, 495, 538, 530, 505, 497, 511, 496, 521, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[557, 467, 513, 516, 551, 506, 529, 499, 2, 0, 0, 0, 0, 0, 0, 0, 500, 541, 517, 534, 524, 531, 458, 494, 0, 0, 0, 0, 0, 0, 0, 0, 494, 548, 500, 511, 499, 535, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 500, 501, 523, 509, 470, 526, 9, 0, 0, 0, 0, 0, 0, 0, 487, 549, 560, 519, 536, 479, 502, 510, 7, 0, 0, 0, 0, 0, 0, 0, 534, 524, 510, 501, 470, 493, 512, 527, 10, 3, 0, 0, 0, 0, 0, 0, 494, 543, 469, 508, 513, 510, 505, 526, 20, 0, 0, 0, 0, 0, 0, 0, 529, 513, 507, 465, 535, 515, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4099, 4090, 4073, 4149, 4084, 4088, 4045] avg=4096.0000 max=4149 max_rank=4 min=4045 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 534, 506, 514, 536, 506, 497, 506, 2, 0, 0, 0, 0, 0, 0, 0, 496, 492, 499, 500, 542, 516, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 531, 570, 493, 512, 492, 541, 505, 529, 0, 0, 0, 0, 0, 0, 0, 0, 485, 448, 544, 537, 536, 510, 522, 515, 9, 0, 0, 0, 0, 0, 0, 0, 503, 507, 498, 508, 523, 470, 481, 520, 7, 0, 0, 0, 0, 0, 0, 0, 531, 508, 507, 542, 492, 498, 538, 517, 10, 3, 0, 0, 0, 0, 0, 0, 507, 513, 517, 526, 496, 518, 501, 507, 21, 0, 0, 0, 0, 0, 0, 0, 511, 533, 523, 520, 502, 505, 490, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4062, 4173, 4106, 4017, 4146, 4106, 4055] avg=4096.0000 max=4173 max_rank=2 min=4017 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 537, 486, 518, 526, 499, 495, 485, 2, 0, 0, 0, 0, 0, 0, 0, 543, 509, 503, 499, 487, 488, 525, 537, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 527, 486, 527, 508, 499, 534, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 514, 469, 528, 486, 522, 509, 10, 0, 0, 0, 0, 0, 0, 0, 509, 518, 508, 495, 525, 509, 512, 521, 7, 0, 0, 0, 0, 0, 0, 0, 510, 490, 507, 542, 486, 526, 512, 495, 11, 4, 0, 0, 0, 0, 0, 0, 485, 494, 493, 516, 525, 510, 549, 520, 21, 0, 0, 0, 0, 0, 0, 0, 472, 508, 530, 527, 522, 527, 505, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4091, 4114, 4064, 4104, 4083, 4113, 4134] avg=4096.0000 max=4134 max_rank=7 min=4064 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[471, 516, 502, 528, 498, 488, 505, 502, 2, 0, 0, 0, 0, 0, 0, 0, 509, 539, 547, 546, 500, 537, 522, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 475, 491, 521, 530, 494, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 477, 519, 510, 516, 516, 512, 9, 0, 0, 0, 0, 0, 0, 0, 517, 533, 534, 522, 496, 524, 459, 538, 7, 0, 0, 0, 0, 0, 0, 0, 509, 506, 495, 531, 511, 527, 451, 520, 10, 3, 0, 0, 0, 0, 0, 0, 550, 509, 496, 508, 490, 517, 564, 478, 20, 0, 0, 0, 0, 0, 0, 0, 556, 546, 478, 497, 521, 535, 475, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4223, 4052, 4054, 4130, 4063, 4132, 4102] avg=4096.0000 max=4223 max_rank=1 min=4012 min_rank=0 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 503, 480, 556, 509, 535, 531, 513, 2, 0, 0, 0, 0, 0, 0, 0, 516, 473, 509, 467, 495, 534, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 487, 523, 524, 551, 536, 544, 508, 509, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 477, 514, 482, 500, 536, 508, 10, 0, 0, 0, 0, 0, 0, 0, 505, 521, 509, 504, 469, 524, 500, 516, 7, 0, 0, 0, 0, 0, 0, 0, 482, 505, 545, 497, 514, 532, 533, 463, 11, 4, 0, 0, 0, 0, 0, 0, 535, 515, 511, 505, 530, 509, 513, 468, 22, 0, 0, 0, 0, 0, 0, 0, 502, 513, 463, 541, 508, 543, 532, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4025, 4182, 4063, 4055, 4086, 4108, 4109] avg=4096.0000 max=4182 max_rank=2 min=4025 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 515, 504, 500, 509, 513, 515, 535, 8, 7, 0, 0, 0, 0, 0, 0, 494, 527, 509, 508, 517, 495, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 541, 488, 464, 488, 477, 524, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 546, 565, 522, 543, 502, 491, 491, 518, 12, 0, 0, 0, 0, 0, 0, 0, 520, 532, 505, 509, 484, 494, 506, 514, 10, 0, 0, 0, 0, 0, 0, 0, 507, 487, 500, 484, 490, 561, 524, 524, 14, 8, 0, 0, 0, 0, 0, 0, 468, 503, 508, 537, 518, 503, 504, 469, 24, 0, 0, 0, 0, 0, 0, 0, 557, 503, 535, 541, 510, 524, 463, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4111, 4018, 4190, 4074, 4099, 4034, 4125] avg=4096.0000 max=4190 max_rank=3 min=4018 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[486, 510, 512, 514, 494, 512, 540, 508, 2, 0, 0, 0, 0, 0, 0, 0, 465, 519, 494, 480, 487, 520, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 509, 530, 489, 525, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 522, 536, 493, 536, 514, 564, 495, 499, 9, 0, 0, 0, 0, 0, 0, 0, 488, 546, 493, 542, 518, 494, 508, 515, 7, 0, 0, 0, 0, 0, 0, 0, 510, 531, 527, 497, 505, 489, 536, 497, 12, 3, 0, 0, 0, 0, 0, 0, 529, 511, 507, 507, 529, 493, 494, 527, 23, 0, 0, 0, 0, 0, 0, 0, 494, 488, 521, 517, 496, 517, 543, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 3989, 4093, 4168, 4111, 4107, 4120, 4102] avg=4096.0000 max=4168 max_rank=3 min=3989 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[567, 525, 496, 543, 504, 487, 498, 491, 2, 0, 0, 0, 0, 0, 0, 0, 504, 547, 524, 526, 533, 521, 527, 486, 0, 0, 0, 0, 0, 0, 0, 0, 500, 504, 500, 526, 532, 486, 502, 496, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 497, 488, 511, 513, 512, 502, 9, 0, 0, 0, 0, 0, 0, 0, 509, 519, 531, 519, 515, 527, 491, 510, 7, 0, 0, 0, 0, 0, 0, 0, 503, 512, 503, 517, 505, 509, 525, 513, 10, 3, 0, 0, 0, 0, 0, 0, 518, 501, 513, 520, 462, 507, 530, 495, 21, 0, 0, 0, 0, 0, 0, 0, 538, 530, 505, 497, 511, 496, 490, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4168, 4046, 4050, 4128, 4100, 4067, 4096] avg=4096.0000 max=4168 max_rank=1 min=4046 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4352, 4224, 4224, 4096, 4096, 3968, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[504, 481, 506, 491, 449, 521, 450, 482, 545, 505, 514, 523, 503, 514, 536, 511, 495, 527, 523, 533, 484, 542, 505, 501, 494, 552, 497, 555, 494, 507, 502, 502, 504, 501, 528, 553, 506, 496, 482, 488, 498, 516, 523, 521, 529, 517, 513, 490, 584, 516, 559, 521, 499, 468, 478, 527, 520, 544, 557, 517, 548, 546, 483, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[483, 503, 519, 500, 533, 528, 533, 523, 492, 503, 510, 519, 501, 518, 502, 515, 451, 503, 529, 485, 500, 531, 498, 516, 520, 532, 485, 471, 522, 521, 522, 462, 506, 518, 524, 523, 535, 493, 513, 466, 528, 484, 528, 534, 523, 509, 504, 526, 513, 506, 520, 550, 508, 551, 519, 511, 564, 495, 522, 490, 534, 514, 513, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[527, 473, 515, 498, 490, 547, 536, 550, 501, 484, 526, 511, 516, 541, 513, 484, 441, 499, 512, 523, 544, 536, 536, 541, 510, 527, 515, 529, 522, 503, 519, 513, 560, 506, 517, 535, 472, 514, 520, 509, 504, 467, 533, 477, 522, 484, 471, 487, 493, 524, 488, 502, 543, 496, 511, 508, 501, 530, 498, 533, 548, 502, 525, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[527, 549, 513, 522, 524, 522, 510, 474, 489, 512, 540, 557, 521, 545, 529, 511, 500, 497, 502, 539, 520, 505, 507, 485, 518, 542, 527, 482, 471, 524, 507, 496, 525, 501, 501, 516, 548, 501, 453, 543, 496, 479, 508, 488, 526, 512, 517, 522, 519, 512, 482, 500, 486, 524, 552, 500, 541, 510, 545, 508, 507, 512, 472, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[485, 496, 466, 504, 496, 503, 522, 536, 522, 534, 508, 506, 517, 506, 485, 507, 491, 529, 534, 510, 548, 540, 519, 457, 514, 513, 495, 511, 530, 548, 515, 493, 500, 538, 470, 516, 481, 547, 516, 523, 482, 531, 541, 522, 558, 477, 515, 515, 494, 508, 517, 518, 547, 531, 489, 522, 520, 485, 526, 497, 536, 497, 491, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[484, 502, 488, 510, 513, 489, 522, 549, 540, 526, 528, 507, 531, 495, 531, 507, 529, 504, 492, 495, 520, 517, 503, 546, 504, 565, 483, 507, 503, 534, 521, 506, 515, 500, 528, 514, 498, 535, 493, 492, 529, 493, 536, 497, 532, 511, 453, 543, 530, 518, 478, 507, 502, 478, 529, 501, 516, 528, 506, 503, 516, 490, 523, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[541, 525, 481, 533, 501, 511, 495, 520, 555, 518, 502, 503, 512, 481, 505, 528, 506, 505, 493, 512, 483, 508, 533, 522, 517, 502, 537, 514, 507, 548, 497, 514, 495, 500, 490, 523, 503, 521, 516, 527, 533, 522, 533, 498, 522, 517, 522, 502, 503, 523, 482, 519, 473, 519, 534, 490, 517, 518, 510, 537, 504, 482, 498, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[479, 458, 511, 516, 526, 488, 537, 496, 500, 513, 536, 503, 548, 516, 522, 534, 508, 530, 537, 570, 505, 513, 501, 505, 500, 493, 484, 487, 490, 511, 514, 500, 510, 501, 504, 522, 511, 497, 546, 534, 544, 503, 512, 515, 542, 498, 510, 520, 473, 504, 501, 519, 509, 524, 515, 568, 509, 508, 480, 490, 518, 565, 480, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[504, 481, 506, 491, 449, 521, 450, 482, 30, 6, 0, 0, 0, 0, 0, 0, 545, 505, 484, 523, 503, 514, 536, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 527, 523, 533, 484, 542, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 494, 552, 497, 555, 494, 507, 502, 502, 8, 0, 0, 0, 0, 0, 0, 0, 504, 501, 528, 553, 506, 496, 482, 488, 3, 1, 0, 0, 0, 0, 0, 0, 498, 516, 523, 521, 529, 517, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 584, 516, 559, 518, 499, 468, 478, 527, 0, 0, 0, 0, 0, 0, 0, 0, 520, 544, 557, 517, 533, 546, 483, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3920, 4121, 4110, 4111, 4062, 4107, 4149, 4188] avg=4096.0000 max=4188 max_rank=7 min=3920 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[527, 549, 513, 522, 524, 522, 510, 474, 32, 6, 0, 0, 0, 0, 0, 0, 489, 512, 508, 557, 521, 545, 529, 511, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 502, 539, 520, 505, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 518, 542, 527, 482, 471, 524, 507, 496, 8, 0, 0, 0, 0, 0, 0, 0, 525, 501, 501, 516, 548, 501, 453, 543, 3, 1, 0, 0, 0, 0, 0, 0, 496, 479, 508, 488, 526, 512, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 482, 497, 486, 524, 552, 500, 0, 0, 0, 0, 0, 0, 0, 0, 541, 510, 545, 508, 492, 512, 472, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4172, 4055, 4075, 4092, 4048, 4072, 4075] avg=4096.0000 max=4179 max_rank=0 min=4048 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 496, 466, 504, 496, 503, 522, 536, 30, 6, 0, 0, 0, 0, 0, 0, 522, 534, 478, 506, 517, 506, 485, 507, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 534, 510, 548, 540, 519, 457, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 495, 511, 530, 548, 515, 493, 8, 0, 0, 0, 0, 0, 0, 0, 500, 538, 470, 516, 481, 547, 516, 523, 3, 1, 0, 0, 0, 0, 0, 0, 482, 531, 541, 522, 558, 477, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 517, 515, 547, 531, 489, 522, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 526, 497, 521, 497, 491, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4055, 4128, 4127, 4095, 4141, 4123, 4055] avg=4096.0000 max=4141 max_rank=5 min=4044 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[483, 503, 519, 500, 533, 528, 533, 523, 30, 6, 0, 0, 0, 0, 0, 0, 492, 503, 480, 519, 501, 518, 502, 515, 0, 0, 0, 0, 0, 0, 0, 0, 451, 503, 529, 485, 500, 531, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 520, 532, 485, 471, 522, 521, 522, 462, 8, 0, 0, 0, 0, 0, 0, 0, 506, 518, 524, 523, 535, 493, 513, 466, 3, 1, 0, 0, 0, 0, 0, 0, 528, 484, 528, 534, 523, 509, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 520, 547, 508, 551, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 564, 495, 522, 490, 519, 514, 513, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4030, 4013, 4043, 4082, 4136, 4175, 4131] avg=4096.0000 max=4175 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[527, 473, 515, 498, 490, 547, 536, 550, 31, 6, 0, 0, 0, 0, 0, 0, 501, 484, 495, 511, 516, 541, 513, 484, 0, 0, 0, 0, 0, 0, 0, 0, 441, 499, 512, 523, 544, 536, 536, 541, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 515, 529, 522, 503, 519, 513, 8, 0, 0, 0, 0, 0, 0, 0, 560, 506, 517, 535, 472, 514, 520, 509, 3, 1, 0, 0, 0, 0, 0, 0, 504, 467, 533, 477, 522, 484, 471, 487, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 488, 499, 543, 496, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 498, 533, 533, 502, 525, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4045, 4132, 4146, 4137, 3945, 4062, 4128] avg=4096.0000 max=4173 max_rank=0 min=3945 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 458, 511, 516, 526, 488, 537, 496, 32, 6, 0, 0, 0, 0, 0, 0, 500, 513, 504, 503, 548, 516, 522, 534, 0, 0, 0, 0, 0, 0, 0, 0, 508, 530, 537, 570, 505, 513, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 493, 484, 487, 490, 511, 514, 500, 8, 0, 0, 0, 0, 0, 0, 0, 510, 501, 504, 522, 511, 497, 546, 534, 3, 1, 0, 0, 0, 0, 0, 0, 544, 503, 512, 515, 542, 498, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 473, 504, 501, 516, 509, 524, 515, 568, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 480, 490, 503, 565, 480, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4140, 4169, 3987, 4129, 4144, 4110, 4040] avg=4096.0000 max=4169 max_rank=2 min=3987 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[541, 525, 481, 533, 501, 511, 495, 520, 33, 11, 0, 0, 0, 0, 0, 0, 555, 518, 469, 503, 512, 481, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 493, 512, 483, 501, 533, 522, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 537, 514, 507, 548, 497, 514, 12, 0, 0, 0, 0, 0, 0, 0, 495, 500, 490, 523, 503, 521, 516, 527, 6, 8, 3, 0, 0, 0, 0, 0, 533, 522, 533, 498, 522, 517, 522, 502, 4, 0, 0, 0, 0, 0, 0, 0, 503, 523, 482, 513, 473, 519, 534, 490, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 510, 537, 473, 482, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4071, 4055, 4148, 4092, 4153, 4037, 4061] avg=4096.0000 max=4153 max_rank=5 min=4037 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[484, 502, 488, 510, 513, 489, 522, 549, 31, 6, 0, 0, 0, 0, 0, 0, 540, 526, 497, 507, 531, 495, 531, 507, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 492, 495, 520, 517, 503, 546, 0, 0, 0, 0, 0, 0, 0, 0, 504, 565, 483, 507, 503, 534, 521, 506, 8, 0, 0, 0, 0, 0, 0, 0, 515, 500, 528, 514, 498, 535, 493, 492, 3, 1, 0, 0, 0, 0, 0, 0, 529, 493, 536, 497, 532, 511, 453, 543, 0, 0, 0, 0, 0, 0, 0, 0, 530, 518, 478, 504, 502, 478, 529, 501, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 506, 503, 501, 490, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4134, 4106, 4131, 4079, 4094, 4040, 4090] avg=4096.0000 max=4134 max_rank=1 min=4040 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[513, 550, 500, 503, 537, 532, 536, 503, 493, 487, 506, 484, 477, 505, 510, 509, 528, 503, 469, 489, 500, 500, 494, 507, 585, 524, 503, 510, 528, 555, 523, 542, 510, 528, 519, 544, 486, 506, 545, 489, 524, 489, 498, 535, 520, 519, 520, 500, 542, 509, 509, 529, 504, 519, 516, 494, 480, 484, 523, 511, 504, 498, 490, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[549, 501, 521, 520, 556, 489, 505, 521, 494, 499, 496, 499, 480, 481, 533, 509, 539, 523, 525, 553, 501, 475, 520, 561, 522, 497, 502, 491, 542, 515, 538, 490, 492, 496, 508, 492, 565, 523, 521, 541, 536, 515, 498, 485, 517, 510, 560, 500, 501, 514, 483, 539, 488, 526, 520, 518, 524, 500, 547, 499, 494, 464, 482, 463] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[507, 511, 572, 519, 538, 526, 513, 530, 531, 461, 511, 499, 529, 478, 489, 541, 518, 554, 527, 516, 489, 519, 483, 537, 504, 481, 492, 510, 481, 509, 504, 534, 503, 504, 550, 495, 514, 501, 498, 487, 483, 513, 481, 528, 503, 509, 520, 536, 555, 571, 492, 554, 536, 526, 536, 518, 491, 485, 525, 487, 494, 464, 514, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[492, 515, 540, 546, 476, 521, 503, 464, 489, 493, 523, 480, 527, 548, 493, 495, 508, 529, 514, 488, 592, 505, 525, 513, 536, 500, 517, 498, 515, 498, 544, 489, 532, 539, 545, 553, 521, 472, 500, 525, 534, 517, 503, 487, 532, 532, 516, 494, 485, 489, 535, 484, 497, 545, 514, 494, 515, 487, 498, 503, 496, 519, 515, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[510, 506, 535, 512, 528, 511, 534, 545, 524, 519, 499, 502, 505, 507, 519, 512, 518, 520, 506, 544, 513, 508, 478, 535, 507, 527, 513, 529, 500, 504, 530, 502, 501, 507, 492, 531, 536, 486, 505, 496, 507, 538, 517, 476, 533, 504, 518, 510, 514, 491, 527, 507, 510, 525, 523, 474, 520, 496, 558, 454, 525, 495, 497, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[523, 541, 510, 515, 512, 504, 528, 527, 510, 536, 496, 492, 512, 515, 504, 498, 514, 519, 497, 506, 557, 510, 491, 526, 511, 530, 496, 511, 535, 522, 525, 529, 519, 516, 466, 556, 563, 517, 503, 531, 513, 525, 503, 489, 493, 484, 493, 487, 503, 521, 491, 519, 512, 520, 506, 555, 525, 491, 509, 466, 497, 487, 501, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[518, 529, 500, 504, 488, 517, 466, 511, 515, 533, 546, 514, 519, 488, 492, 510, 492, 523, 503, 504, 525, 468, 524, 491, 504, 513, 482, 529, 517, 490, 504, 543, 490, 479, 549, 545, 509, 554, 538, 513, 508, 536, 538, 549, 484, 554, 519, 487, 503, 533, 536, 480, 535, 482, 556, 508, 510, 513, 523, 467, 488, 528, 493, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[517, 528, 491, 520, 498, 531, 516, 493, 507, 525, 516, 525, 499, 476, 535, 490, 530, 522, 519, 472, 490, 520, 491, 534, 487, 596, 533, 514, 513, 522, 527, 500, 519, 499, 522, 549, 518, 521, 532, 503, 523, 480, 528, 518, 488, 486, 509, 491, 523, 490, 556, 481, 538, 465, 523, 509, 507, 485, 523, 508, 497, 518, 509, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 502, 510, 515, 512, 504, 528, 527, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 496, 492, 512, 515, 504, 498, 22, 21, 2, 0, 0, 0, 0, 0, 514, 519, 497, 506, 557, 510, 491, 516, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 496, 511, 535, 522, 494, 529, 0, 0, 0, 0, 0, 0, 0, 0, 519, 516, 466, 517, 563, 517, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 503, 489, 493, 484, 493, 487, 8, 0, 0, 0, 0, 0, 0, 0, 503, 521, 491, 519, 512, 520, 484, 555, 0, 0, 0, 0, 0, 0, 0, 0, 525, 491, 509, 466, 497, 487, 501, 505, 39, 39, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4108, 4110, 4128, 4132, 3995, 4105, 4069] avg=4096.0000 max=4132 max_rank=4 min=3995 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 510, 500, 503, 537, 532, 536, 503, 0, 0, 0, 0, 0, 0, 0, 0, 493, 487, 506, 484, 477, 505, 510, 509, 23, 21, 2, 0, 0, 0, 0, 0, 528, 503, 469, 489, 500, 500, 494, 498, 0, 0, 0, 0, 0, 0, 0, 0, 585, 524, 503, 510, 528, 555, 492, 542, 0, 0, 0, 0, 0, 0, 0, 0, 510, 528, 519, 506, 486, 506, 545, 489, 0, 0, 0, 0, 0, 0, 0, 0, 524, 489, 498, 535, 520, 519, 520, 500, 7, 0, 0, 0, 0, 0, 0, 0, 542, 509, 509, 529, 504, 519, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 480, 484, 523, 511, 504, 498, 490, 519, 40, 38, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4017, 3981, 4239, 4089, 4112, 4099, 4097] avg=4096.0000 max=4239 max_rank=3 min=3981 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 469, 535, 512, 528, 511, 534, 545, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 499, 502, 505, 507, 519, 512, 23, 21, 2, 0, 0, 0, 0, 0, 518, 520, 506, 544, 513, 508, 478, 525, 0, 0, 0, 0, 0, 0, 0, 0, 507, 527, 513, 529, 500, 504, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 492, 494, 536, 486, 505, 496, 0, 0, 0, 0, 0, 0, 0, 0, 507, 538, 517, 476, 533, 504, 518, 510, 8, 0, 0, 0, 0, 0, 0, 0, 514, 491, 527, 507, 510, 525, 500, 474, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 558, 454, 525, 495, 497, 493, 37, 37, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4133, 4112, 4081, 4017, 4111, 4048, 4122] avg=4096.0000 max=4144 max_rank=0 min=4017 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[549, 464, 521, 520, 556, 489, 505, 521, 0, 0, 0, 0, 0, 0, 0, 0, 494, 499, 496, 499, 480, 481, 533, 509, 23, 21, 2, 0, 0, 0, 0, 0, 539, 523, 525, 553, 501, 475, 520, 551, 0, 0, 0, 0, 0, 0, 0, 0, 522, 497, 502, 491, 542, 515, 506, 490, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 508, 458, 565, 523, 521, 541, 0, 0, 0, 0, 0, 0, 0, 0, 536, 515, 498, 485, 517, 510, 560, 500, 8, 0, 0, 0, 0, 0, 0, 0, 501, 514, 483, 539, 488, 526, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 524, 500, 547, 499, 494, 464, 482, 463, 37, 34, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4037, 4187, 4065, 4104, 4129, 4066, 4055] avg=4096.0000 max=4187 max_rank=2 min=4037 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 474, 572, 519, 538, 526, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 531, 461, 511, 499, 529, 478, 489, 541, 24, 20, 2, 0, 0, 0, 0, 0, 518, 554, 527, 516, 489, 519, 483, 527, 0, 0, 0, 0, 0, 0, 0, 0, 504, 481, 492, 510, 481, 509, 474, 534, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 550, 461, 514, 501, 498, 487, 0, 0, 0, 0, 0, 0, 0, 0, 483, 513, 481, 528, 503, 509, 520, 536, 8, 0, 0, 0, 0, 0, 0, 0, 555, 571, 492, 554, 536, 526, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 525, 487, 494, 464, 514, 482, 37, 34, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4085, 4133, 3985, 4018, 4081, 4264, 4023] avg=4096.0000 max=4264 max_rank=6 min=3985 min_rank=3 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[492, 474, 540, 546, 476, 521, 503, 464, 0, 0, 0, 0, 0, 0, 0, 0, 489, 493, 523, 480, 527, 548, 493, 495, 27, 24, 6, 0, 0, 0, 0, 0, 508, 529, 514, 488, 592, 505, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 536, 500, 517, 498, 515, 498, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 532, 539, 545, 510, 521, 472, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 503, 487, 532, 532, 516, 494, 12, 0, 0, 0, 0, 0, 0, 0, 485, 489, 535, 484, 497, 545, 487, 494, 0, 0, 0, 0, 0, 0, 0, 0, 515, 487, 498, 503, 496, 519, 515, 514, 41, 43, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4105, 4156, 4057, 4144, 4127, 4016, 4147] avg=4096.0000 max=4156 max_rank=2 min=4016 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 489, 491, 520, 498, 531, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 516, 525, 499, 476, 535, 490, 23, 21, 2, 0, 0, 0, 0, 0, 530, 522, 519, 472, 490, 520, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 487, 596, 533, 514, 513, 522, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 522, 511, 518, 521, 532, 503, 0, 0, 0, 0, 0, 0, 0, 0, 523, 480, 528, 518, 488, 486, 509, 491, 8, 0, 0, 0, 0, 0, 0, 0, 523, 490, 556, 481, 538, 465, 500, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 485, 523, 508, 497, 518, 509, 513, 39, 38, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4119, 4068, 4161, 4125, 4031, 4062, 4147] avg=4096.0000 max=4161 max_rank=3 min=4031 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 500, 504, 488, 517, 466, 511, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 546, 514, 519, 488, 492, 510, 25, 20, 2, 0, 0, 0, 0, 0, 492, 523, 503, 504, 525, 468, 524, 482, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 482, 529, 517, 490, 474, 543, 0, 0, 0, 0, 0, 0, 0, 0, 490, 479, 549, 507, 509, 554, 538, 513, 0, 0, 0, 0, 0, 0, 0, 0, 508, 536, 538, 549, 484, 554, 519, 487, 7, 0, 0, 0, 0, 0, 0, 0, 503, 533, 536, 480, 535, 482, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 523, 467, 488, 528, 493, 499, 39, 38, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4164, 4021, 4052, 4139, 4182, 4108, 4108] avg=4096.0000 max=4182 max_rank=5 min=3994 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 3968, 4224, 3968, 4096, 4096, 4096, 4096, 384, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[515, 535, 531, 479, 547, 537, 514, 496, 508, 528, 492, 496, 500, 473, 497, 501, 530, 511, 480, 483, 514, 518, 502, 526, 523, 493, 526, 499, 553, 517, 502, 529, 498, 504, 494, 495, 556, 524, 525, 484, 517, 533, 500, 521, 506, 529, 552, 521, 537, 491, 501, 526, 499, 495, 529, 526, 534, 511, 532, 501, 457, 479, 512, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[458, 506, 505, 533, 510, 511, 476, 509, 523, 532, 515, 506, 538, 513, 548, 527, 538, 527, 472, 499, 503, 525, 514, 502, 492, 521, 541, 522, 512, 498, 516, 533, 516, 524, 482, 525, 526, 500, 471, 519, 537, 511, 546, 473, 534, 527, 540, 526, 521, 535, 477, 538, 520, 496, 523, 535, 480, 495, 461, 512, 510, 501, 494, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[505, 506, 543, 507, 556, 501, 517, 497, 541, 479, 496, 535, 524, 536, 496, 541, 496, 514, 499, 477, 518, 494, 481, 472, 536, 509, 502, 549, 546, 508, 519, 475, 512, 533, 471, 536, 507, 539, 485, 535, 490, 507, 512, 526, 555, 514, 497, 499, 485, 488, 534, 526, 519, 510, 504, 507, 496, 531, 502, 527, 524, 525, 481, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[493, 534, 517, 489, 498, 512, 523, 512, 510, 506, 491, 494, 486, 549, 477, 473, 521, 526, 523, 541, 514, 488, 516, 523, 549, 546, 497, 495, 502, 531, 477, 514, 501, 490, 496, 504, 545, 482, 496, 512, 544, 526, 450, 490, 510, 545, 507, 515, 485, 564, 504, 514, 534, 545, 538, 516, 534, 501, 509, 526, 520, 523, 502, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[487, 532, 504, 528, 521, 517, 518, 511, 460, 557, 515, 507, 494, 512, 530, 537, 506, 506, 500, 584, 543, 516, 517, 472, 487, 507, 489, 519, 544, 510, 484, 523, 516, 541, 521, 483, 466, 528, 494, 527, 480, 528, 533, 517, 527, 507, 484, 502, 501, 492, 523, 522, 506, 503, 556, 515, 536, 522, 510, 514, 494, 477, 520, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[530, 507, 504, 569, 490, 517, 519, 517, 524, 479, 516, 471, 504, 518, 487, 519, 550, 482, 532, 567, 486, 527, 494, 518, 513, 483, 499, 525, 517, 502, 517, 515, 493, 513, 500, 495, 546, 488, 492, 475, 532, 555, 504, 527, 554, 504, 496, 507, 480, 486, 527, 546, 531, 512, 506, 521, 517, 508, 465, 544, 527, 500, 502, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[544, 545, 538, 530, 527, 518, 541, 470, 488, 527, 518, 552, 517, 497, 475, 543, 543, 508, 496, 555, 507, 522, 487, 495, 511, 485, 490, 519, 518, 531, 536, 518, 534, 511, 520, 526, 511, 511, 504, 533, 511, 490, 518, 523, 494, 528, 529, 488, 503, 450, 522, 497, 536, 493, 482, 494, 492, 474, 536, 534, 481, 476, 522, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[493, 507, 498, 509, 515, 539, 500, 548, 501, 515, 545, 555, 521, 478, 534, 472, 519, 482, 494, 521, 552, 509, 524, 500, 487, 507, 518, 543, 522, 515, 512, 493, 520, 476, 494, 493, 490, 501, 517, 496, 529, 513, 522, 528, 492, 514, 461, 533, 510, 520, 519, 531, 523, 495, 516, 552, 505, 542, 495, 475, 537, 499, 523, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 511, 531, 479, 547, 537, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 492, 496, 500, 473, 497, 501, 9, 0, 0, 0, 0, 0, 0, 0, 530, 511, 480, 483, 514, 518, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 523, 493, 526, 499, 541, 517, 502, 529, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 494, 495, 556, 524, 525, 484, 15, 14, 3, 0, 0, 0, 0, 0, 517, 533, 500, 521, 480, 529, 552, 521, 0, 0, 0, 0, 0, 0, 0, 0, 537, 491, 501, 511, 499, 495, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 532, 501, 457, 479, 512, 524, 26, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4004, 4064, 4130, 4112, 4153, 4089, 4086] avg=4096.0000 max=4153 max_rank=5 min=4004 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[458, 484, 505, 533, 510, 511, 476, 509, 0, 0, 0, 0, 0, 0, 0, 0, 523, 532, 515, 506, 538, 513, 548, 527, 8, 0, 0, 0, 0, 0, 0, 0, 538, 527, 472, 499, 503, 525, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, 541, 522, 501, 498, 516, 533, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 482, 525, 526, 500, 471, 519, 15, 13, 3, 0, 0, 0, 0, 0, 537, 511, 546, 473, 506, 527, 540, 526, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 477, 523, 520, 496, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 480, 495, 461, 512, 510, 501, 494, 488, 28, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4210, 4080, 4124, 4094, 4166, 4130, 3978] avg=4096.0000 max=4210 max_rank=1 min=3978 min_rank=7 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[505, 484, 543, 507, 556, 501, 517, 497, 0, 0, 0, 0, 0, 0, 0, 0, 541, 479, 496, 535, 524, 536, 496, 541, 8, 0, 0, 0, 0, 0, 0, 0, 496, 514, 499, 477, 518, 494, 481, 472, 0, 0, 0, 0, 0, 0, 0, 0, 536, 509, 502, 549, 535, 508, 519, 475, 0, 0, 0, 0, 0, 0, 0, 0, 512, 533, 471, 536, 507, 539, 485, 535, 15, 13, 3, 0, 0, 0, 0, 0, 490, 507, 512, 526, 526, 514, 497, 499, 0, 0, 0, 0, 0, 0, 0, 0, 485, 488, 534, 511, 519, 510, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 502, 527, 524, 525, 481, 516, 29, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4156, 3951, 4133, 4149, 4071, 4058, 4140] avg=4096.0000 max=4156 max_rank=1 min=3951 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 510, 517, 489, 498, 512, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 491, 494, 486, 549, 477, 473, 8, 0, 0, 0, 0, 0, 0, 0, 521, 526, 523, 541, 514, 488, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0, 549, 546, 497, 495, 491, 531, 477, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 490, 496, 504, 545, 482, 496, 512, 15, 14, 3, 0, 0, 0, 0, 0, 544, 526, 450, 490, 483, 545, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 485, 564, 504, 499, 534, 545, 538, 516, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 509, 526, 520, 523, 502, 513, 27, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 3994, 4152, 4100, 4058, 4060, 4185, 4165] avg=4096.0000 max=4185 max_rank=6 min=3994 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 508, 504, 528, 521, 517, 518, 511, 0, 0, 0, 0, 0, 0, 0, 0, 460, 557, 515, 507, 494, 512, 530, 537, 8, 0, 0, 0, 0, 0, 0, 0, 506, 506, 500, 584, 543, 516, 517, 472, 0, 0, 0, 0, 0, 0, 0, 0, 487, 507, 489, 519, 533, 510, 484, 523, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 521, 483, 466, 528, 494, 527, 15, 14, 3, 0, 0, 0, 0, 0, 480, 528, 533, 517, 499, 507, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 492, 523, 507, 506, 503, 556, 515, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 510, 514, 494, 477, 520, 486, 28, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4120, 4144, 4052, 4108, 4050, 4103, 4097] avg=4096.0000 max=4144 max_rank=2 min=4050 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 485, 504, 569, 490, 517, 519, 517, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 516, 471, 504, 518, 487, 519, 8, 0, 0, 0, 0, 0, 0, 0, 550, 482, 532, 567, 486, 527, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 499, 525, 506, 502, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 500, 495, 546, 488, 492, 475, 16, 13, 3, 0, 0, 0, 0, 0, 532, 555, 504, 527, 525, 504, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 480, 486, 527, 530, 531, 512, 506, 521, 0, 0, 0, 0, 0, 0, 0, 0, 517, 508, 465, 544, 527, 500, 502, 517, 29, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4026, 4156, 4060, 4034, 4150, 4093, 4118] avg=4096.0000 max=4156 max_rank=2 min=4026 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[544, 508, 538, 530, 527, 518, 541, 470, 0, 0, 0, 0, 0, 0, 0, 0, 488, 527, 518, 552, 517, 497, 475, 543, 12, 0, 0, 0, 0, 0, 0, 0, 543, 508, 496, 555, 507, 522, 487, 495, 7, 0, 0, 0, 0, 0, 0, 0, 511, 485, 490, 519, 493, 531, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 520, 526, 511, 511, 504, 533, 18, 21, 6, 0, 0, 0, 0, 0, 511, 490, 518, 523, 465, 528, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 503, 450, 522, 479, 536, 493, 482, 494, 0, 0, 0, 0, 0, 0, 0, 0, 492, 474, 536, 534, 481, 476, 522, 494, 29, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4129, 4120, 4083, 4195, 4052, 3959, 4054] avg=4096.0000 max=4195 max_rank=4 min=3959 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 485, 498, 509, 515, 539, 500, 548, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 545, 555, 521, 478, 534, 472, 8, 0, 0, 0, 0, 0, 0, 0, 519, 482, 494, 521, 552, 509, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 487, 507, 518, 543, 511, 515, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 520, 476, 494, 493, 490, 501, 517, 496, 15, 13, 3, 0, 0, 0, 0, 0, 529, 513, 522, 528, 466, 514, 461, 533, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 519, 516, 523, 495, 516, 552, 0, 0, 0, 0, 0, 0, 0, 0, 505, 542, 495, 475, 537, 499, 523, 519, 26, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4129, 4101, 4086, 4018, 4066, 4151, 4130] avg=4096.0000 max=4151 max_rank=6 min=4018 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[509, 529, 491, 529, 526, 528, 475, 503, 495, 516, 523, 482, 533, 495, 487, 505, 500, 494, 507, 548, 534, 488, 561, 502, 497, 569, 543, 520, 507, 523, 517, 508, 537, 492, 499, 531, 533, 519, 505, 509, 501, 550, 493, 521, 477, 482, 503, 509, 502, 499, 524, 554, 501, 492, 489, 495, 530, 525, 538, 490, 503, 484, 500, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[501, 501, 496, 530, 537, 482, 525, 517, 495, 501, 517, 490, 490, 521, 547, 461, 500, 504, 533, 546, 543, 528, 491, 498, 494, 504, 512, 488, 486, 507, 519, 481, 507, 514, 543, 546, 463, 509, 507, 538, 550, 538, 505, 528, 507, 506, 524, 499, 528, 499, 557, 494, 527, 504, 486, 531, 499, 530, 514, 540, 485, 543, 493, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[501, 517, 504, 487, 513, 519, 493, 529, 487, 552, 527, 504, 504, 516, 494, 476, 502, 532, 511, 464, 496, 496, 529, 517, 525, 503, 522, 565, 538, 505, 496, 526, 488, 523, 506, 479, 496, 498, 527, 509, 530, 517, 541, 502, 513, 511, 534, 508, 533, 538, 537, 494, 492, 537, 512, 540, 503, 498, 495, 522, 521, 497, 495, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[512, 475, 499, 508, 489, 509, 537, 531, 519, 496, 508, 522, 496, 505, 525, 496, 533, 496, 499, 502, 518, 526, 524, 487, 540, 511, 515, 511, 484, 534, 517, 476, 530, 521, 517, 538, 496, 523, 523, 506, 504, 502, 515, 515, 519, 493, 509, 469, 555, 520, 543, 533, 503, 515, 515, 512, 495, 508, 529, 542, 494, 531, 497, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[492, 527, 458, 522, 538, 513, 520, 532, 489, 521, 547, 530, 532, 542, 491, 513, 532, 496, 485, 475, 498, 521, 514, 473, 506, 531, 468, 509, 492, 510, 529, 526, 529, 513, 507, 526, 550, 493, 548, 494, 558, 515, 520, 537, 504, 518, 526, 492, 515, 477, 509, 523, 479, 490, 534, 543, 486, 501, 527, 530, 502, 482, 508, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[523, 536, 488, 517, 532, 496, 485, 498, 500, 502, 529, 520, 528, 517, 461, 518, 543, 543, 523, 541, 500, 523, 485, 528, 473, 516, 541, 521, 504, 484, 556, 491, 518, 539, 506, 477, 547, 526, 442, 515, 502, 515, 516, 462, 490, 554, 517, 534, 514, 514, 516, 528, 484, 507, 455, 521, 512, 535, 497, 507, 508, 538, 541, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[495, 492, 508, 490, 499, 487, 486, 504, 516, 548, 510, 517, 559, 527, 527, 511, 477, 511, 494, 505, 540, 540, 517, 510, 531, 550, 482, 502, 518, 507, 536, 497, 518, 489, 481, 533, 476, 523, 514, 502, 555, 528, 523, 505, 537, 492, 502, 496, 508, 512, 525, 509, 500, 547, 501, 491, 503, 499, 529, 512, 494, 537, 511, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[569, 472, 522, 500, 529, 509, 528, 525, 531, 493, 527, 474, 538, 510, 497, 496, 496, 492, 511, 536, 520, 525, 537, 509, 514, 515, 526, 538, 547, 503, 521, 524, 524, 513, 479, 511, 514, 475, 500, 465, 493, 497, 490, 515, 519, 522, 500, 517, 519, 511, 528, 523, 525, 525, 518, 516, 485, 502, 519, 491, 518, 526, 492, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 529, 491, 529, 526, 528, 475, 503, 19, 1, 0, 0, 0, 0, 0, 0, 495, 516, 523, 482, 533, 495, 487, 505, 5, 2, 0, 0, 0, 0, 0, 0, 500, 494, 507, 548, 534, 488, 556, 502, 0, 0, 0, 0, 0, 0, 0, 0, 497, 551, 543, 520, 507, 523, 517, 508, 0, 0, 0, 0, 0, 0, 0, 0, 537, 492, 499, 531, 533, 519, 505, 509, 12, 0, 0, 0, 0, 0, 0, 0, 495, 550, 493, 521, 477, 482, 503, 509, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 505, 554, 501, 492, 489, 495, 0, 0, 0, 0, 0, 0, 0, 0, 530, 525, 538, 490, 503, 484, 500, 537, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4043, 4129, 4166, 4137, 4030, 4037, 4116] avg=4096.0000 max=4166 max_rank=3 min=4030 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 501, 496, 530, 537, 482, 525, 517, 21, 1, 0, 0, 0, 0, 0, 0, 495, 501, 517, 490, 490, 521, 547, 461, 4, 2, 0, 0, 0, 0, 0, 0, 500, 504, 533, 546, 543, 528, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 494, 489, 512, 488, 486, 507, 519, 481, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 543, 546, 463, 509, 507, 538, 10, 0, 0, 0, 0, 0, 0, 0, 544, 538, 505, 528, 507, 506, 524, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 499, 536, 494, 527, 504, 486, 531, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 514, 540, 485, 543, 493, 509, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4028, 4139, 3976, 4137, 4151, 4105, 4121] avg=4096.0000 max=4151 max_rank=5 min=3976 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[523, 536, 488, 517, 532, 496, 485, 498, 19, 1, 0, 0, 0, 0, 0, 0, 500, 502, 529, 520, 528, 517, 461, 518, 4, 2, 0, 0, 0, 0, 0, 0, 543, 543, 523, 541, 500, 523, 481, 528, 0, 0, 0, 0, 0, 0, 0, 0, 473, 501, 541, 521, 504, 484, 556, 491, 0, 0, 0, 0, 0, 0, 0, 0, 518, 539, 506, 477, 547, 526, 442, 515, 10, 0, 0, 0, 0, 0, 0, 0, 496, 515, 516, 462, 490, 554, 517, 534, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 497, 528, 484, 507, 455, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 535, 497, 507, 508, 538, 541, 509, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4081, 4182, 4071, 4080, 4084, 4020, 4155] avg=4096.0000 max=4182 max_rank=2 min=4020 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[492, 527, 458, 522, 538, 513, 520, 532, 19, 1, 0, 0, 0, 0, 0, 0, 489, 521, 547, 530, 532, 542, 491, 513, 5, 2, 0, 0, 0, 0, 0, 0, 532, 496, 485, 475, 498, 521, 509, 473, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 468, 509, 492, 510, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 529, 513, 507, 526, 550, 493, 548, 494, 11, 0, 0, 0, 0, 0, 0, 0, 552, 515, 520, 537, 504, 518, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 515, 477, 490, 523, 479, 490, 534, 543, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 527, 530, 502, 482, 508, 500, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4172, 3989, 4054, 4171, 4164, 4051, 4045] avg=4096.0000 max=4172 max_rank=1 min=3989 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 475, 499, 508, 489, 509, 537, 531, 20, 1, 0, 0, 0, 0, 0, 0, 519, 496, 508, 522, 496, 505, 525, 496, 5, 2, 0, 0, 0, 0, 0, 0, 533, 496, 499, 502, 518, 526, 519, 487, 0, 0, 0, 0, 0, 0, 0, 0, 540, 496, 515, 511, 484, 534, 517, 476, 0, 0, 0, 0, 0, 0, 0, 0, 530, 521, 517, 538, 496, 523, 523, 506, 10, 0, 0, 0, 0, 0, 0, 0, 498, 502, 515, 515, 519, 493, 509, 469, 0, 0, 0, 0, 0, 0, 0, 0, 555, 520, 523, 533, 503, 515, 515, 512, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 529, 542, 494, 531, 497, 496, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4074, 4080, 4073, 4164, 4020, 4176, 4100] avg=4096.0000 max=4176 max_rank=6 min=4020 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 517, 504, 487, 513, 519, 493, 529, 24, 7, 0, 0, 0, 0, 0, 0, 487, 552, 527, 504, 504, 516, 494, 476, 8, 7, 0, 0, 0, 0, 0, 0, 502, 532, 511, 464, 496, 496, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 525, 472, 522, 565, 538, 505, 496, 526, 0, 0, 0, 0, 0, 0, 0, 0, 488, 523, 506, 479, 496, 498, 527, 509, 15, 0, 0, 0, 0, 0, 0, 0, 514, 517, 541, 502, 513, 511, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 513, 494, 492, 537, 512, 540, 0, 0, 0, 0, 0, 0, 0, 0, 503, 498, 495, 522, 521, 497, 495, 522, 9, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4075, 4039, 4149, 4041, 4140, 4159, 4071] avg=4096.0000 max=4159 max_rank=6 min=4039 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 492, 508, 490, 499, 487, 486, 504, 19, 1, 0, 0, 0, 0, 0, 0, 516, 548, 510, 517, 559, 527, 527, 511, 5, 2, 0, 0, 0, 0, 0, 0, 477, 511, 494, 505, 540, 540, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 531, 533, 482, 502, 518, 507, 536, 497, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 481, 533, 476, 523, 514, 502, 11, 0, 0, 0, 0, 0, 0, 0, 549, 528, 523, 505, 537, 492, 502, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 506, 509, 500, 547, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 503, 499, 529, 512, 494, 537, 511, 523, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3981, 4222, 4089, 4106, 4047, 4132, 4074, 4117] avg=4096.0000 max=4222 max_rank=1 min=3981 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[569, 472, 522, 500, 529, 509, 528, 525, 19, 1, 0, 0, 0, 0, 0, 0, 531, 493, 527, 474, 538, 510, 497, 496, 5, 2, 0, 0, 0, 0, 0, 0, 496, 492, 511, 536, 520, 525, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 526, 538, 547, 503, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 479, 511, 514, 475, 500, 465, 10, 0, 0, 0, 0, 0, 0, 0, 487, 497, 490, 515, 519, 522, 500, 517, 0, 0, 0, 0, 0, 0, 0, 0, 519, 511, 509, 523, 525, 525, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 485, 502, 519, 491, 518, 526, 492, 502, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4073, 4121, 4173, 3991, 4047, 4146, 4043] avg=4096.0000 max=4174 max_rank=0 min=3991 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[493, 534, 510, 539, 501, 527, 520, 553, 564, 506, 504, 503, 483, 510, 539, 473, 527, 496, 491, 492, 513, 542, 529, 509, 498, 508, 510, 505, 510, 483, 497, 512, 514, 486, 520, 519, 518, 493, 499, 480, 510, 492, 496, 522, 503, 496, 528, 509, 521, 563, 518, 537, 480, 486, 520, 511, 524, 488, 530, 515, 583, 526, 497, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[523, 525, 518, 479, 488, 497, 489, 512, 499, 501, 476, 530, 490, 520, 511, 530, 486, 498, 530, 487, 504, 508, 544, 546, 504, 524, 526, 508, 515, 566, 504, 506, 529, 507, 488, 536, 528, 505, 503, 538, 514, 529, 509, 468, 514, 475, 500, 526, 518, 546, 521, 485, 517, 514, 508, 526, 519, 471, 501, 569, 477, 536, 537, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[568, 496, 517, 553, 496, 510, 508, 515, 540, 517, 549, 494, 522, 478, 537, 507, 514, 537, 511, 519, 495, 526, 502, 492, 495, 502, 541, 489, 539, 528, 530, 483, 469, 524, 510, 527, 493, 470, 520, 524, 525, 516, 517, 504, 487, 482, 500, 493, 525, 482, 522, 513, 510, 541, 529, 491, 549, 471, 539, 511, 493, 527, 497, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[512, 494, 557, 521, 511, 543, 525, 504, 502, 522, 497, 522, 483, 526, 484, 506, 522, 493, 479, 486, 545, 541, 513, 508, 520, 501, 477, 525, 530, 487, 507, 496, 508, 539, 516, 503, 535, 511, 517, 543, 506, 528, 490, 504, 490, 533, 538, 497, 516, 487, 500, 490, 491, 524, 546, 500, 493, 479, 503, 518, 532, 516, 543, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[493, 506, 517, 510, 531, 491, 493, 511, 502, 494, 528, 535, 514, 496, 554, 482, 527, 508, 528, 522, 529, 495, 508, 520, 493, 532, 533, 510, 526, 535, 478, 490, 512, 510, 515, 515, 519, 514, 497, 538, 523, 507, 531, 498, 507, 454, 525, 514, 510, 496, 560, 503, 513, 509, 505, 527, 491, 527, 517, 504, 491, 529, 514, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[542, 547, 485, 496, 494, 548, 508, 507, 507, 481, 489, 482, 532, 479, 495, 508, 494, 508, 492, 530, 548, 514, 535, 511, 502, 479, 505, 524, 524, 492, 491, 530, 464, 520, 495, 534, 516, 563, 527, 530, 501, 495, 518, 513, 520, 530, 521, 478, 535, 505, 528, 532, 497, 526, 517, 502, 521, 505, 526, 520, 523, 523, 520, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[528, 528, 503, 520, 544, 499, 512, 486, 519, 507, 490, 471, 533, 503, 515, 513, 521, 517, 493, 520, 511, 510, 491, 529, 527, 540, 513, 550, 513, 534, 521, 499, 501, 524, 504, 489, 545, 525, 521, 516, 517, 471, 517, 526, 530, 531, 545, 505, 494, 504, 471, 500, 525, 454, 514, 499, 518, 502, 504, 498, 486, 546, 501, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[510, 478, 527, 496, 477, 508, 525, 509, 507, 519, 555, 483, 547, 538, 491, 524, 497, 498, 546, 476, 549, 507, 540, 528, 525, 497, 531, 506, 523, 515, 528, 523, 508, 517, 505, 504, 492, 512, 509, 528, 501, 510, 497, 508, 533, 486, 501, 509, 525, 504, 529, 487, 495, 535, 512, 546, 496, 503, 487, 489, 511, 545, 485, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[482, 534, 510, 539, 501, 527, 520, 553, 0, 0, 0, 0, 0, 0, 0, 0, 564, 506, 504, 503, 483, 510, 539, 473, 5, 5, 3, 3, 0, 0, 0, 0, 527, 496, 491, 492, 499, 542, 529, 509, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 510, 505, 505, 483, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 520, 519, 518, 493, 499, 472, 0, 0, 0, 0, 0, 0, 0, 0, 510, 492, 496, 522, 503, 496, 528, 509, 14, 11, 3, 0, 0, 0, 0, 0, 521, 563, 518, 537, 480, 486, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 524, 488, 530, 515, 583, 523, 497, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4098, 4085, 4018, 4021, 4084, 4133, 4163] avg=4096.0000 max=4166 max_rank=0 min=4018 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[555, 496, 517, 553, 496, 510, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 540, 517, 549, 494, 522, 478, 537, 507, 6, 5, 3, 3, 0, 0, 0, 0, 514, 537, 511, 519, 481, 526, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 495, 502, 541, 489, 533, 528, 530, 483, 0, 0, 0, 0, 0, 0, 0, 0, 469, 524, 510, 527, 493, 470, 520, 516, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 517, 504, 487, 482, 500, 493, 14, 13, 3, 0, 0, 0, 0, 0, 525, 482, 522, 513, 510, 541, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 549, 471, 539, 511, 493, 524, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4161, 4082, 4101, 4029, 4054, 4110, 4081] avg=4096.0000 max=4161 max_rank=1 min=4029 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 494, 557, 521, 511, 543, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 502, 522, 497, 522, 483, 526, 484, 506, 5, 5, 3, 3, 0, 0, 0, 0, 522, 493, 479, 486, 530, 541, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 477, 525, 525, 487, 507, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 539, 516, 503, 535, 511, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 490, 504, 490, 533, 538, 497, 15, 12, 3, 0, 0, 0, 0, 0, 516, 487, 500, 490, 491, 524, 543, 500, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 503, 518, 532, 513, 543, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4058, 4072, 4038, 4164, 4116, 4051, 4114] avg=4096.0000 max=4164 max_rank=4 min=4038 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 525, 518, 479, 488, 497, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 476, 530, 490, 520, 511, 530, 5, 5, 3, 3, 0, 0, 0, 0, 486, 498, 530, 487, 490, 508, 544, 546, 0, 0, 0, 0, 0, 0, 0, 0, 504, 524, 526, 508, 510, 566, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 488, 536, 528, 505, 503, 530, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 509, 468, 514, 475, 500, 526, 14, 12, 3, 0, 0, 0, 0, 0, 518, 546, 521, 485, 517, 514, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 471, 501, 569, 477, 533, 537, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4073, 4089, 4148, 4126, 4064, 4132, 4117] avg=4096.0000 max=4148 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 478, 527, 496, 477, 508, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 555, 483, 547, 538, 491, 524, 5, 5, 3, 3, 0, 0, 0, 0, 497, 498, 546, 476, 534, 507, 540, 528, 0, 0, 0, 0, 0, 0, 0, 0, 525, 497, 531, 506, 518, 515, 528, 523, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 505, 504, 492, 512, 509, 520, 0, 0, 0, 0, 0, 0, 0, 0, 501, 510, 497, 508, 533, 486, 501, 509, 15, 12, 3, 0, 0, 0, 0, 0, 525, 504, 529, 487, 495, 535, 509, 546, 0, 0, 0, 0, 0, 0, 0, 0, 496, 503, 487, 489, 511, 542, 485, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4180, 4126, 4143, 4067, 4075, 4130, 4029] avg=4096.0000 max=4180 max_rank=1 min=4018 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[516, 528, 503, 520, 544, 499, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 490, 471, 533, 503, 515, 513, 5, 5, 3, 3, 0, 0, 0, 0, 521, 517, 493, 520, 497, 510, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0, 527, 540, 513, 550, 508, 534, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 501, 524, 504, 489, 545, 525, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 517, 471, 517, 526, 530, 531, 545, 505, 14, 12, 3, 0, 0, 0, 0, 0, 494, 504, 471, 500, 525, 454, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 504, 498, 486, 543, 501, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4067, 4078, 4192, 4117, 4171, 3958, 4077] avg=4096.0000 max=4192 max_rank=3 min=3958 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[530, 547, 485, 496, 494, 548, 508, 507, 0, 0, 0, 0, 0, 0, 0, 0, 507, 481, 489, 482, 532, 479, 495, 508, 5, 5, 3, 3, 0, 0, 0, 0, 494, 508, 492, 530, 533, 514, 535, 511, 0, 0, 0, 0, 0, 0, 0, 0, 502, 479, 505, 524, 519, 492, 491, 530, 0, 0, 0, 0, 0, 0, 0, 0, 464, 520, 495, 534, 516, 563, 527, 522, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 518, 513, 520, 530, 521, 478, 15, 12, 3, 0, 0, 0, 0, 0, 535, 505, 528, 532, 497, 526, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 526, 520, 523, 520, 520, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 3989, 4117, 4042, 4141, 4106, 4139, 4119] avg=4096.0000 max=4141 max_rank=4 min=3989 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[478, 506, 517, 510, 531, 491, 493, 511, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 528, 535, 514, 496, 554, 482, 11, 11, 8, 7, 0, 0, 0, 0, 527, 508, 528, 522, 511, 495, 508, 520, 0, 0, 0, 0, 0, 0, 0, 0, 493, 532, 533, 510, 515, 535, 478, 490, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 515, 515, 519, 514, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 531, 498, 507, 454, 525, 514, 18, 15, 6, 0, 0, 0, 0, 0, 510, 496, 560, 503, 513, 509, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 517, 504, 491, 522, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4142, 4119, 4086, 4103, 4098, 4115, 4068] avg=4096.0000 max=4142 max_rank=1 min=4037 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 15:43:11] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 444.1 | throughput per GPU (TFLOP/s/GPU): 278.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097754E+01 | loss scale: 1.0 | grad norm: 30.504 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[493, 476, 508, 502, 505, 493, 489, 525, 491, 482, 513, 527, 514, 519, 517, 519, 503, 564, 525, 520, 529, 483, 530, 505, 476, 518, 517, 521, 493, 519, 525, 540, 514, 510, 526, 499, 505, 523, 515, 492, 510, 525, 530, 521, 547, 513, 525, 513, 531, 482, 527, 513, 502, 509, 505, 487, 511, 484, 497, 524, 513, 527, 541, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[460, 523, 495, 549, 527, 473, 502, 488, 493, 518, 510, 491, 551, 499, 489, 519, 510, 501, 526, 505, 539, 470, 479, 521, 509, 514, 484, 535, 511, 507, 522, 591, 496, 502, 493, 487, 550, 510, 531, 501, 540, 514, 472, 562, 520, 543, 487, 517, 461, 494, 475, 526, 519, 519, 558, 528, 572, 496, 481, 500, 523, 519, 530, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[524, 456, 538, 530, 533, 553, 486, 544, 540, 490, 540, 509, 519, 506, 548, 498, 492, 528, 528, 503, 508, 496, 539, 511, 517, 504, 505, 522, 513, 552, 511, 488, 486, 477, 516, 512, 518, 501, 482, 502, 535, 488, 505, 503, 521, 533, 502, 557, 491, 481, 540, 488, 494, 509, 463, 553, 471, 502, 486, 509, 515, 523, 538, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[487, 493, 509, 515, 497, 507, 511, 517, 522, 524, 462, 509, 498, 492, 512, 500, 503, 504, 485, 518, 493, 525, 561, 506, 481, 477, 467, 574, 502, 465, 519, 501, 481, 508, 504, 493, 557, 509, 525, 526, 498, 512, 488, 503, 549, 542, 544, 506, 533, 531, 515, 499, 553, 534, 496, 514, 463, 553, 498, 527, 563, 531, 547, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[510, 516, 504, 545, 510, 487, 524, 513, 546, 510, 506, 512, 503, 490, 491, 521, 501, 513, 487, 493, 543, 519, 524, 491, 519, 520, 516, 493, 484, 510, 513, 494, 517, 531, 560, 504, 486, 510, 503, 493, 476, 545, 523, 509, 528, 519, 531, 505, 515, 527, 520, 513, 485, 523, 506, 480, 543, 513, 539, 539, 514, 484, 517, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[498, 478, 496, 493, 542, 512, 510, 464, 509, 513, 517, 542, 454, 553, 527, 512, 522, 507, 508, 549, 493, 522, 545, 513, 498, 544, 473, 517, 527, 539, 509, 528, 517, 519, 520, 477, 488, 523, 522, 496, 536, 494, 501, 523, 535, 501, 517, 553, 506, 507, 544, 492, 504, 490, 514, 478, 491, 513, 513, 479, 549, 528, 517, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[503, 544, 506, 539, 511, 552, 514, 511, 487, 496, 517, 508, 525, 510, 505, 496, 516, 482, 506, 482, 498, 509, 523, 523, 485, 490, 505, 476, 509, 511, 531, 523, 508, 531, 550, 518, 541, 523, 549, 539, 518, 496, 490, 484, 502, 536, 546, 535, 513, 523, 494, 537, 498, 477, 481, 497, 525, 544, 498, 504, 494, 525, 480, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[483, 512, 499, 513, 538, 496, 559, 520, 502, 521, 501, 531, 508, 522, 537, 508, 511, 502, 498, 541, 529, 528, 499, 502, 512, 536, 501, 515, 524, 521, 517, 469, 476, 562, 498, 496, 494, 497, 497, 550, 492, 527, 493, 522, 528, 514, 526, 491, 503, 483, 531, 507, 476, 496, 524, 501, 523, 515, 557, 511, 484, 521, 498, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 476, 508, 502, 505, 493, 489, 525, 12, 7, 0, 0, 0, 0, 0, 0, 491, 482, 513, 527, 514, 519, 517, 519, 8, 0, 0, 0, 0, 0, 0, 0, 503, 564, 525, 520, 529, 483, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0, 476, 518, 517, 521, 493, 519, 525, 540, 10, 0, 0, 0, 0, 0, 0, 0, 514, 510, 526, 499, 505, 523, 515, 492, 2, 0, 0, 0, 0, 0, 0, 0, 510, 525, 530, 521, 503, 513, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 531, 482, 527, 513, 502, 509, 505, 487, 37, 0, 0, 0, 0, 0, 0, 0, 511, 484, 497, 524, 513, 527, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4090, 4157, 4119, 4086, 4140, 4093, 4073] avg=4096.0000 max=4157 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[483, 512, 499, 513, 538, 496, 559, 520, 11, 7, 0, 0, 0, 0, 0, 0, 502, 521, 501, 531, 508, 522, 537, 508, 8, 0, 0, 0, 0, 0, 0, 0, 511, 502, 498, 541, 529, 528, 497, 502, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 501, 515, 524, 521, 517, 469, 9, 0, 0, 0, 0, 0, 0, 0, 476, 562, 498, 496, 494, 497, 497, 550, 2, 0, 0, 0, 0, 0, 0, 0, 492, 527, 493, 522, 485, 514, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 531, 507, 476, 496, 524, 501, 36, 0, 0, 0, 0, 0, 0, 0, 523, 515, 557, 511, 484, 521, 470, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4138, 4108, 4104, 4072, 4050, 4057, 4101] avg=4096.0000 max=4138 max_rank=0 min=4050 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 544, 506, 539, 511, 552, 514, 511, 11, 7, 0, 0, 0, 0, 0, 0, 487, 496, 517, 508, 525, 510, 505, 496, 7, 0, 0, 0, 0, 0, 0, 0, 516, 482, 506, 482, 498, 509, 521, 523, 0, 0, 0, 0, 0, 0, 0, 0, 485, 490, 505, 476, 509, 511, 531, 523, 9, 0, 0, 0, 0, 0, 0, 0, 508, 531, 550, 518, 541, 523, 549, 539, 2, 0, 0, 0, 0, 0, 0, 0, 518, 496, 490, 484, 461, 536, 546, 535, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 494, 537, 498, 477, 481, 497, 34, 0, 0, 0, 0, 0, 0, 0, 525, 544, 498, 504, 494, 525, 453, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4051, 4037, 4039, 4261, 4066, 4054, 4062] avg=4096.0000 max=4261 max_rank=4 min=4037 min_rank=2 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[460, 523, 495, 549, 527, 473, 502, 488, 12, 7, 0, 0, 0, 0, 0, 0, 493, 518, 510, 491, 551, 499, 489, 519, 8, 0, 0, 0, 0, 0, 0, 0, 510, 501, 526, 505, 539, 470, 477, 521, 0, 0, 0, 0, 0, 0, 0, 0, 509, 514, 484, 535, 511, 507, 522, 591, 10, 0, 0, 0, 0, 0, 0, 0, 496, 502, 493, 487, 550, 510, 531, 501, 2, 0, 0, 0, 0, 0, 0, 0, 540, 514, 472, 562, 477, 543, 487, 517, 0, 0, 0, 0, 0, 0, 0, 0, 461, 494, 475, 526, 519, 519, 558, 528, 36, 0, 0, 0, 0, 0, 0, 0, 572, 496, 481, 500, 523, 519, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4078, 4049, 4183, 4072, 4112, 4116, 4122] avg=4096.0000 max=4183 max_rank=3 min=4036 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[487, 493, 509, 515, 497, 507, 511, 517, 12, 7, 0, 0, 0, 0, 0, 0, 522, 524, 462, 509, 498, 492, 512, 500, 8, 0, 0, 0, 0, 0, 0, 0, 503, 504, 485, 518, 493, 525, 559, 506, 0, 0, 0, 0, 0, 0, 0, 0, 481, 477, 467, 574, 502, 465, 519, 501, 10, 0, 0, 0, 0, 0, 0, 0, 481, 508, 504, 493, 557, 509, 525, 526, 2, 0, 0, 0, 0, 0, 0, 0, 498, 512, 488, 503, 504, 542, 544, 506, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 515, 499, 553, 534, 496, 514, 38, 0, 0, 0, 0, 0, 0, 0, 463, 553, 498, 527, 563, 531, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4027, 4093, 3996, 4105, 4097, 4213, 4182] avg=4096.0000 max=4213 max_rank=6 min=3996 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[524, 456, 538, 530, 533, 553, 486, 544, 16, 11, 0, 0, 0, 0, 0, 0, 540, 490, 540, 509, 519, 506, 548, 498, 12, 0, 0, 0, 0, 0, 0, 0, 492, 528, 528, 503, 508, 496, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 505, 522, 513, 552, 511, 488, 13, 0, 0, 0, 0, 0, 0, 0, 486, 477, 516, 512, 518, 501, 482, 502, 7, 1, 0, 0, 0, 0, 0, 0, 535, 488, 505, 503, 471, 533, 502, 557, 0, 0, 0, 0, 0, 0, 0, 0, 491, 481, 540, 488, 494, 509, 463, 553, 39, 0, 0, 0, 0, 0, 0, 0, 471, 502, 486, 509, 515, 523, 496, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4162, 4098, 4125, 4002, 4094, 4058, 4038] avg=4096.0000 max=4191 max_rank=0 min=4002 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16192 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 516, 504, 545, 510, 487, 524, 513, 12, 7, 0, 0, 0, 0, 0, 0, 546, 510, 506, 512, 503, 490, 491, 521, 8, 0, 0, 0, 0, 0, 0, 0, 501, 513, 487, 493, 543, 519, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 516, 493, 484, 510, 513, 494, 9, 0, 0, 0, 0, 0, 0, 0, 517, 531, 560, 504, 486, 510, 503, 493, 2, 0, 0, 0, 0, 0, 0, 0, 476, 545, 523, 509, 485, 519, 531, 505, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 520, 513, 485, 523, 506, 480, 36, 0, 0, 0, 0, 0, 0, 0, 543, 513, 539, 539, 514, 484, 488, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4087, 4069, 4058, 4106, 4093, 4105, 4122] avg=4096.0000 max=4128 max_rank=0 min=4058 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 478, 496, 493, 542, 512, 510, 464, 12, 7, 0, 0, 0, 0, 0, 0, 509, 513, 517, 542, 454, 553, 527, 512, 8, 0, 0, 0, 0, 0, 0, 0, 522, 507, 508, 549, 493, 522, 543, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 544, 473, 517, 527, 539, 509, 528, 9, 0, 0, 0, 0, 0, 0, 0, 517, 519, 520, 477, 488, 523, 522, 496, 2, 0, 0, 0, 0, 0, 0, 0, 536, 494, 501, 523, 491, 501, 517, 553, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 544, 492, 504, 490, 514, 478, 37, 0, 0, 0, 0, 0, 0, 0, 491, 513, 513, 479, 549, 528, 488, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4135, 4157, 4144, 4064, 4116, 4072, 4068] avg=4096.0000 max=4157 max_rank=2 min=4012 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[527, 529, 527, 564, 515, 484, 480, 511, 552, 506, 496, 484, 537, 491, 496, 497, 538, 477, 491, 507, 502, 472, 519, 492, 480, 532, 515, 539, 547, 519, 539, 515, 488, 440, 497, 474, 509, 519, 494, 523, 511, 515, 548, 543, 539, 493, 509, 500, 514, 526, 485, 482, 530, 531, 534, 522, 553, 507, 512, 491, 486, 523, 527, 563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[482, 499, 512, 552, 532, 519, 501, 483, 524, 515, 506, 514, 481, 485, 499, 509, 540, 514, 504, 541, 510, 495, 509, 452, 501, 545, 531, 503, 496, 500, 520, 513, 515, 504, 526, 495, 490, 522, 514, 493, 523, 550, 503, 512, 542, 500, 517, 502, 503, 503, 518, 478, 550, 502, 534, 494, 516, 514, 537, 510, 537, 524, 534, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[515, 538, 517, 498, 511, 525, 523, 495, 519, 482, 516, 515, 523, 517, 517, 514, 508, 473, 535, 520, 481, 513, 494, 461, 519, 481, 482, 515, 496, 533, 504, 509, 510, 496, 562, 540, 555, 527, 503, 511, 522, 515, 509, 508, 548, 486, 503, 485, 519, 544, 513, 513, 536, 492, 492, 487, 483, 511, 570, 533, 521, 527, 483, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[469, 495, 548, 503, 503, 510, 528, 535, 500, 508, 526, 488, 524, 523, 544, 484, 515, 514, 524, 518, 491, 528, 492, 508, 561, 524, 521, 520, 516, 503, 510, 526, 504, 482, 544, 511, 520, 516, 451, 519, 503, 500, 498, 483, 515, 480, 514, 528, 539, 518, 558, 527, 527, 500, 473, 497, 496, 543, 487, 544, 509, 471, 511, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[517, 516, 486, 481, 519, 541, 539, 490, 515, 521, 501, 511, 542, 510, 491, 499, 509, 514, 512, 507, 523, 501, 516, 488, 506, 525, 514, 511, 500, 490, 527, 494, 505, 495, 521, 494, 518, 527, 518, 519, 523, 499, 554, 535, 498, 495, 529, 553, 541, 513, 519, 493, 488, 489, 483, 504, 511, 518, 539, 529, 503, 510, 528, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[508, 542, 531, 526, 493, 496, 530, 491, 516, 468, 493, 512, 530, 526, 506, 495, 506, 506, 524, 474, 524, 547, 559, 485, 525, 536, 513, 522, 495, 543, 514, 521, 530, 489, 495, 545, 544, 493, 521, 509, 538, 487, 522, 521, 501, 529, 547, 514, 498, 500, 523, 476, 502, 524, 522, 509, 504, 501, 492, 464, 504, 488, 493, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[516, 496, 494, 515, 550, 476, 512, 571, 512, 508, 504, 479, 479, 529, 526, 490, 498, 486, 509, 526, 537, 504, 499, 522, 499, 516, 515, 507, 504, 497, 525, 525, 516, 549, 496, 494, 519, 532, 492, 481, 525, 506, 522, 512, 508, 520, 536, 491, 513, 509, 538, 559, 458, 551, 560, 518, 516, 492, 541, 486, 463, 525, 481, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[523, 536, 562, 509, 499, 474, 478, 479, 563, 507, 510, 508, 534, 519, 530, 516, 521, 552, 517, 505, 518, 501, 499, 469, 526, 479, 497, 518, 508, 543, 539, 530, 525, 502, 442, 545, 514, 535, 499, 502, 505, 512, 555, 531, 497, 531, 469, 499, 484, 510, 523, 500, 554, 531, 475, 495, 504, 552, 476, 517, 515, 502, 491, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[527, 529, 520, 564, 515, 484, 480, 511, 0, 0, 0, 0, 0, 0, 0, 0, 552, 506, 496, 484, 537, 491, 496, 497, 6, 5, 4, 0, 0, 0, 0, 0, 538, 477, 491, 507, 502, 472, 519, 492, 27, 7, 0, 0, 0, 0, 0, 0, 480, 532, 515, 539, 547, 519, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 488, 440, 497, 474, 509, 519, 494, 523, 18, 1, 0, 0, 0, 0, 0, 0, 511, 515, 523, 543, 539, 493, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 481, 482, 530, 531, 534, 522, 0, 0, 0, 0, 0, 0, 0, 0, 553, 507, 512, 491, 486, 523, 527, 558, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4074, 4032, 4159, 3963, 4133, 4120, 4157] avg=4096.0000 max=4159 max_rank=3 min=3963 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[482, 499, 505, 552, 532, 519, 501, 483, 0, 0, 0, 0, 0, 0, 0, 0, 524, 515, 506, 514, 481, 485, 499, 509, 6, 5, 4, 0, 0, 0, 0, 0, 540, 514, 504, 541, 510, 495, 509, 452, 26, 7, 0, 0, 0, 0, 0, 0, 501, 545, 531, 503, 496, 500, 494, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 526, 495, 490, 522, 514, 493, 16, 1, 0, 0, 0, 0, 0, 0, 523, 550, 480, 512, 542, 500, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 514, 478, 550, 502, 534, 494, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 537, 510, 537, 524, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4048, 4098, 4083, 4076, 4126, 4078, 4186] avg=4096.0000 max=4186 max_rank=7 min=4048 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 542, 524, 526, 493, 496, 530, 491, 0, 0, 0, 0, 0, 0, 0, 0, 516, 468, 493, 512, 530, 526, 506, 495, 6, 5, 4, 0, 0, 0, 0, 0, 506, 506, 524, 474, 524, 547, 559, 485, 25, 7, 0, 0, 0, 0, 0, 0, 525, 536, 513, 522, 495, 543, 489, 521, 0, 0, 0, 0, 0, 0, 0, 0, 530, 489, 495, 545, 544, 493, 521, 509, 17, 1, 0, 0, 0, 0, 0, 0, 538, 487, 498, 521, 501, 529, 547, 514, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 519, 476, 502, 524, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 492, 464, 504, 488, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4061, 4157, 4144, 4144, 4135, 4050, 3967] avg=4096.0000 max=4157 max_rank=2 min=3967 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[469, 495, 541, 503, 503, 510, 528, 535, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 526, 488, 524, 523, 544, 484, 6, 5, 4, 0, 0, 0, 0, 0, 515, 514, 524, 518, 491, 528, 492, 508, 25, 7, 0, 0, 0, 0, 0, 0, 561, 524, 521, 520, 516, 503, 485, 526, 0, 0, 0, 0, 0, 0, 0, 0, 504, 482, 544, 511, 520, 516, 451, 519, 16, 1, 0, 0, 0, 0, 0, 0, 503, 500, 475, 483, 515, 480, 514, 528, 0, 0, 0, 0, 0, 0, 0, 0, 539, 518, 554, 527, 527, 500, 473, 497, 0, 0, 0, 0, 0, 0, 0, 0, 496, 543, 487, 544, 509, 471, 511, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4112, 4122, 4156, 4064, 3998, 4135, 4097] avg=4096.0000 max=4156 max_rank=3 min=3998 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 538, 510, 498, 511, 525, 523, 495, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 516, 515, 523, 517, 517, 514, 6, 5, 4, 0, 0, 0, 0, 0, 508, 473, 535, 520, 481, 513, 494, 461, 25, 7, 0, 0, 0, 0, 0, 0, 519, 481, 482, 515, 496, 533, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0, 510, 496, 562, 540, 555, 527, 503, 511, 16, 1, 0, 0, 0, 0, 0, 0, 522, 515, 486, 508, 548, 486, 503, 485, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 509, 513, 536, 492, 492, 487, 0, 0, 0, 0, 0, 0, 0, 0, 483, 511, 570, 533, 521, 527, 483, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4118, 4017, 4014, 4221, 4053, 4092, 4138] avg=4096.0000 max=4221 max_rank=4 min=4014 min_rank=3 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 536, 555, 509, 499, 474, 478, 479, 0, 0, 0, 0, 0, 0, 0, 0, 563, 507, 510, 508, 534, 519, 530, 516, 6, 5, 4, 0, 0, 0, 0, 0, 521, 552, 517, 505, 518, 501, 499, 469, 27, 8, 0, 0, 0, 0, 0, 0, 526, 479, 497, 518, 508, 543, 512, 530, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 442, 545, 514, 535, 499, 502, 18, 1, 0, 0, 0, 0, 0, 0, 505, 512, 529, 531, 497, 531, 469, 499, 0, 0, 0, 0, 0, 0, 0, 0, 484, 510, 519, 500, 554, 531, 475, 495, 0, 0, 0, 0, 0, 0, 0, 0, 504, 552, 476, 517, 515, 502, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4202, 4117, 4113, 4083, 4073, 4068, 4059] avg=4096.0000 max=4202 max_rank=1 min=4053 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 516, 480, 481, 519, 541, 539, 490, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 501, 511, 542, 510, 491, 499, 5, 5, 4, 0, 0, 0, 0, 0, 509, 514, 512, 507, 523, 501, 516, 488, 26, 8, 0, 0, 0, 0, 0, 0, 506, 525, 514, 511, 500, 490, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 521, 494, 518, 527, 518, 519, 18, 1, 0, 0, 0, 0, 0, 0, 523, 499, 528, 535, 498, 495, 529, 553, 0, 0, 0, 0, 0, 0, 0, 0, 541, 513, 515, 493, 488, 489, 483, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 539, 529, 503, 510, 528, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4104, 4104, 4041, 4116, 4160, 4026, 4134] avg=4096.0000 max=4160 max_rank=5 min=4026 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[516, 496, 484, 515, 550, 476, 512, 571, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 504, 479, 479, 529, 526, 490, 8, 7, 7, 0, 0, 0, 0, 0, 498, 486, 509, 526, 537, 504, 499, 522, 30, 10, 0, 0, 0, 0, 0, 0, 499, 516, 515, 507, 504, 497, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 516, 549, 496, 494, 519, 532, 492, 481, 20, 2, 0, 0, 0, 0, 0, 0, 525, 506, 492, 512, 508, 520, 536, 491, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 531, 559, 458, 551, 560, 518, 0, 0, 0, 0, 0, 0, 0, 0, 516, 492, 541, 486, 463, 525, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4049, 4121, 4058, 4101, 4090, 4199, 4030] avg=4096.0000 max=4199 max_rank=6 min=4030 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 3968, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[532, 527, 524, 503, 533, 489, 508, 518, 548, 510, 474, 511, 487, 509, 495, 546, 497, 499, 533, 514, 525, 493, 503, 505, 511, 502, 503, 528, 524, 482, 499, 499, 491, 536, 515, 471, 512, 544, 515, 496, 516, 518, 550, 515, 519, 501, 509, 525, 508, 520, 485, 517, 518, 520, 512, 505, 532, 491, 521, 541, 487, 530, 504, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[513, 466, 566, 529, 519, 502, 516, 513, 522, 512, 510, 497, 510, 510, 484, 512, 484, 570, 499, 511, 500, 512, 529, 507, 513, 548, 512, 509, 520, 535, 490, 495, 493, 527, 527, 511, 532, 536, 506, 524, 510, 509, 550, 492, 464, 515, 504, 495, 489, 485, 548, 513, 521, 493, 567, 537, 457, 503, 494, 530, 496, 515, 508, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[494, 514, 502, 526, 500, 476, 475, 534, 504, 503, 523, 502, 480, 499, 544, 518, 511, 506, 534, 496, 503, 540, 505, 520, 496, 581, 517, 521, 510, 494, 496, 512, 520, 518, 496, 512, 549, 511, 525, 522, 502, 496, 512, 514, 502, 545, 505, 511, 513, 507, 521, 511, 493, 493, 515, 493, 500, 542, 519, 541, 519, 538, 508, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[463, 493, 473, 500, 489, 524, 537, 530, 537, 485, 486, 470, 527, 550, 519, 516, 499, 467, 479, 529, 540, 492, 512, 526, 500, 525, 482, 542, 548, 533, 539, 518, 525, 470, 517, 476, 524, 531, 502, 521, 535, 515, 518, 482, 546, 502, 500, 485, 532, 498, 533, 505, 500, 499, 542, 544, 526, 513, 497, 511, 520, 537, 521, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[509, 515, 524, 523, 508, 498, 506, 527, 540, 518, 542, 505, 518, 499, 536, 473, 490, 512, 539, 516, 507, 521, 550, 540, 512, 503, 472, 507, 533, 552, 527, 499, 528, 482, 496, 535, 529, 504, 523, 473, 479, 508, 516, 490, 536, 505, 514, 540, 517, 508, 491, 483, 485, 514, 522, 507, 511, 538, 524, 527, 511, 495, 481, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[478, 515, 466, 536, 493, 523, 510, 479, 497, 564, 541, 527, 455, 521, 559, 493, 551, 519, 525, 535, 519, 492, 542, 508, 486, 554, 529, 494, 482, 559, 497, 493, 523, 494, 491, 512, 485, 492, 512, 525, 531, 464, 504, 504, 490, 479, 562, 500, 511, 505, 519, 532, 489, 519, 519, 535, 505, 538, 504, 543, 494, 511, 531, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[494, 531, 544, 524, 527, 473, 540, 568, 484, 497, 499, 545, 511, 545, 490, 527, 511, 519, 489, 534, 518, 505, 521, 516, 494, 496, 516, 535, 496, 506, 536, 544, 476, 497, 527, 501, 516, 514, 491, 538, 490, 519, 518, 498, 512, 534, 533, 517, 502, 534, 482, 528, 453, 514, 517, 513, 481, 485, 493, 534, 505, 510, 498, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[529, 510, 521, 502, 508, 522, 504, 492, 506, 507, 496, 476, 535, 534, 515, 515, 560, 548, 483, 532, 513, 547, 512, 517, 510, 525, 512, 475, 504, 472, 527, 532, 539, 478, 526, 483, 495, 523, 519, 514, 514, 517, 518, 527, 545, 484, 501, 518, 497, 502, 538, 509, 492, 500, 515, 517, 505, 560, 526, 484, 504, 508, 498, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[532, 527, 524, 503, 533, 489, 508, 518, 9, 0, 0, 0, 0, 0, 0, 0, 548, 510, 474, 511, 487, 509, 495, 546, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 533, 514, 525, 493, 474, 505, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 503, 528, 524, 482, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 491, 536, 515, 471, 512, 544, 515, 496, 8, 0, 0, 0, 0, 0, 0, 0, 516, 518, 550, 515, 519, 501, 509, 525, 4, 0, 0, 0, 0, 0, 0, 0, 508, 520, 485, 517, 518, 520, 512, 505, 6, 0, 0, 0, 0, 0, 0, 0, 532, 491, 521, 541, 487, 530, 504, 513, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4080, 4040, 4038, 4088, 4157, 4091, 4131] avg=4096.0000 max=4157 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[463, 493, 473, 500, 489, 524, 537, 530, 9, 0, 0, 0, 0, 0, 0, 0, 537, 485, 486, 470, 527, 550, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 467, 479, 529, 540, 492, 482, 526, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 482, 542, 548, 533, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 470, 517, 476, 524, 531, 502, 521, 8, 0, 0, 0, 0, 0, 0, 0, 535, 515, 518, 482, 546, 502, 500, 485, 4, 0, 0, 0, 0, 0, 0, 0, 532, 498, 533, 505, 500, 499, 542, 544, 7, 0, 0, 0, 0, 0, 0, 0, 526, 513, 497, 511, 520, 537, 521, 511, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4090, 4014, 4176, 4074, 4087, 4160, 4149] avg=4096.0000 max=4176 max_rank=3 min=4014 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 466, 566, 529, 519, 502, 516, 513, 10, 0, 0, 0, 0, 0, 0, 0, 522, 512, 510, 497, 510, 510, 484, 512, 0, 0, 0, 0, 0, 0, 0, 0, 484, 570, 499, 511, 500, 512, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 512, 509, 520, 535, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 493, 527, 527, 511, 532, 536, 506, 524, 9, 0, 0, 0, 0, 0, 0, 0, 510, 509, 550, 492, 464, 515, 504, 495, 4, 0, 0, 0, 0, 0, 0, 0, 489, 485, 548, 513, 521, 493, 567, 537, 7, 0, 0, 0, 0, 0, 0, 0, 457, 503, 494, 530, 496, 515, 508, 502, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4057, 4080, 4111, 4165, 4043, 4160, 4018] avg=4096.0000 max=4165 max_rank=4 min=4018 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 514, 502, 526, 500, 476, 475, 534, 9, 0, 0, 0, 0, 0, 0, 0, 504, 503, 523, 502, 480, 499, 544, 518, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 534, 496, 503, 540, 476, 520, 0, 0, 0, 0, 0, 0, 0, 0, 496, 569, 517, 521, 510, 494, 496, 512, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 496, 512, 549, 511, 525, 522, 8, 0, 0, 0, 0, 0, 0, 0, 502, 496, 512, 514, 502, 545, 505, 511, 5, 0, 0, 0, 0, 0, 0, 0, 513, 507, 521, 511, 493, 493, 515, 493, 7, 0, 0, 0, 0, 0, 0, 0, 500, 542, 519, 541, 519, 538, 508, 479, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4073, 4086, 4115, 4161, 4092, 4053, 4158] avg=4096.0000 max=4161 max_rank=4 min=4030 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[478, 515, 466, 536, 493, 523, 510, 479, 10, 0, 0, 0, 0, 0, 0, 0, 497, 564, 541, 527, 455, 521, 559, 493, 0, 0, 0, 0, 0, 0, 0, 0, 551, 519, 525, 535, 519, 492, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 486, 543, 529, 494, 482, 559, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 491, 512, 485, 492, 512, 525, 9, 0, 0, 0, 0, 0, 0, 0, 531, 464, 504, 504, 490, 479, 562, 500, 4, 0, 0, 0, 0, 0, 0, 0, 511, 505, 519, 532, 489, 519, 519, 535, 7, 0, 0, 0, 0, 0, 0, 0, 505, 538, 504, 543, 494, 511, 531, 503, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4157, 4159, 4083, 4043, 4038, 4136, 4142] avg=4096.0000 max=4159 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 515, 524, 523, 508, 498, 506, 527, 10, 0, 0, 0, 0, 0, 0, 0, 540, 518, 542, 505, 518, 499, 536, 473, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 539, 516, 507, 521, 518, 540, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 472, 507, 533, 552, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 496, 535, 529, 504, 523, 473, 9, 0, 0, 0, 0, 0, 0, 0, 479, 508, 516, 490, 536, 505, 514, 540, 4, 0, 0, 0, 0, 0, 0, 0, 517, 508, 491, 483, 485, 514, 522, 507, 6, 0, 0, 0, 0, 0, 0, 0, 511, 538, 524, 527, 511, 495, 481, 475, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4131, 4143, 4095, 4079, 4092, 4033, 4075] avg=4096.0000 max=4143 max_rank=2 min=4033 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 531, 544, 524, 527, 473, 540, 568, 10, 0, 0, 0, 0, 0, 0, 0, 484, 497, 499, 545, 511, 545, 490, 527, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 489, 534, 518, 505, 490, 516, 0, 0, 0, 0, 0, 0, 0, 0, 494, 486, 516, 535, 496, 506, 536, 544, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 527, 501, 516, 514, 491, 538, 8, 0, 0, 0, 0, 0, 0, 0, 490, 519, 518, 498, 512, 534, 533, 517, 4, 0, 0, 0, 0, 0, 0, 0, 502, 534, 482, 528, 453, 514, 517, 513, 6, 0, 0, 0, 0, 0, 0, 0, 481, 485, 493, 534, 505, 510, 498, 503, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4098, 4082, 4113, 4068, 4125, 4049, 4022] avg=4096.0000 max=4211 max_rank=0 min=4022 min_rank=7 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 510, 521, 502, 508, 522, 504, 492, 14, 0, 0, 0, 0, 0, 0, 0, 506, 507, 496, 476, 535, 532, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 560, 548, 483, 532, 513, 547, 464, 517, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 512, 475, 504, 472, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 539, 478, 526, 483, 495, 523, 519, 514, 13, 0, 0, 0, 0, 0, 0, 0, 514, 517, 518, 527, 545, 484, 501, 518, 8, 2, 0, 0, 0, 0, 0, 0, 497, 502, 538, 509, 492, 500, 515, 517, 12, 4, 0, 0, 0, 0, 0, 0, 505, 560, 526, 484, 504, 508, 498, 471, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4082, 4164, 4037, 4090, 4134, 4086, 4073] avg=4096.0000 max=4164 max_rank=2 min=4037 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 3968, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[522, 493, 494, 519, 548, 493, 511, 471, 513, 517, 520, 519, 483, 523, 500, 517, 523, 510, 515, 500, 565, 513, 504, 504, 490, 524, 513, 519, 486, 523, 498, 506, 520, 502, 529, 524, 510, 511, 478, 500, 481, 537, 509, 504, 514, 509, 516, 517, 522, 481, 543, 525, 503, 504, 542, 497, 510, 533, 527, 530, 552, 537, 482, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[527, 520, 533, 524, 518, 494, 534, 514, 486, 538, 551, 473, 496, 508, 533, 498, 485, 501, 524, 508, 485, 489, 519, 497, 518, 496, 521, 474, 501, 509, 529, 538, 545, 506, 511, 494, 536, 510, 499, 555, 524, 518, 520, 523, 498, 516, 485, 512, 509, 502, 534, 493, 455, 515, 553, 515, 499, 521, 527, 542, 522, 490, 523, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[501, 530, 517, 500, 521, 474, 495, 497, 462, 554, 495, 520, 500, 517, 516, 502, 547, 513, 513, 492, 487, 502, 500, 539, 541, 523, 524, 504, 491, 536, 521, 502, 533, 506, 501, 499, 534, 489, 506, 474, 527, 517, 548, 528, 515, 480, 490, 548, 523, 508, 517, 559, 531, 531, 509, 540, 482, 496, 506, 527, 508, 501, 511, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[524, 468, 540, 543, 526, 525, 519, 499, 483, 518, 509, 505, 523, 544, 530, 526, 514, 547, 511, 517, 489, 494, 533, 514, 521, 536, 527, 534, 526, 470, 532, 481, 519, 505, 496, 538, 535, 478, 522, 488, 523, 522, 530, 540, 478, 495, 507, 502, 486, 530, 512, 493, 492, 525, 488, 487, 513, 492, 483, 499, 484, 563, 499, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[496, 497, 532, 508, 501, 503, 502, 527, 555, 523, 485, 475, 518, 487, 556, 499, 503, 527, 519, 522, 512, 499, 504, 496, 506, 527, 512, 515, 547, 511, 495, 537, 491, 529, 502, 505, 546, 523, 451, 554, 547, 504, 529, 534, 524, 480, 504, 540, 501, 537, 498, 524, 494, 497, 554, 500, 489, 517, 517, 485, 529, 494, 487, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[511, 523, 539, 520, 468, 529, 557, 539, 525, 478, 539, 500, 511, 487, 550, 511, 512, 508, 508, 519, 506, 469, 517, 528, 542, 486, 487, 505, 509, 522, 490, 506, 499, 542, 518, 505, 527, 529, 482, 533, 504, 480, 499, 517, 526, 483, 505, 500, 496, 536, 507, 504, 498, 531, 525, 496, 516, 478, 519, 519, 518, 525, 531, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[513, 525, 514, 509, 528, 542, 526, 526, 517, 488, 540, 515, 503, 537, 505, 526, 472, 507, 487, 543, 524, 505, 533, 495, 478, 477, 464, 515, 504, 535, 519, 500, 536, 520, 497, 505, 486, 517, 497, 523, 492, 505, 521, 514, 551, 532, 539, 523, 542, 491, 485, 523, 507, 468, 542, 496, 478, 548, 484, 533, 483, 483, 562, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[517, 520, 540, 501, 487, 444, 526, 545, 525, 504, 535, 518, 504, 490, 496, 495, 510, 515, 513, 515, 540, 464, 522, 559, 469, 518, 473, 549, 539, 500, 473, 520, 487, 528, 530, 503, 538, 509, 521, 482, 490, 521, 514, 534, 568, 546, 493, 530, 510, 526, 505, 519, 534, 484, 513, 504, 459, 507, 518, 498, 528, 510, 525, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[524, 468, 523, 543, 526, 525, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 509, 505, 523, 544, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 514, 547, 511, 517, 489, 494, 533, 514, 6, 1, 0, 0, 0, 0, 0, 0, 521, 536, 527, 534, 526, 470, 532, 481, 11, 4, 0, 0, 0, 0, 0, 0, 519, 505, 496, 538, 535, 478, 522, 488, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 530, 513, 478, 495, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 512, 493, 492, 525, 488, 487, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 483, 499, 484, 563, 499, 516, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4137, 4126, 4142, 4081, 4070, 4013, 4072] avg=4096.0000 max=4142 max_rank=3 min=4013 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16236 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 530, 500, 500, 521, 474, 495, 497, 0, 0, 0, 0, 0, 0, 0, 0, 462, 554, 495, 520, 500, 517, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 547, 513, 513, 492, 487, 502, 500, 539, 6, 0, 0, 0, 0, 0, 0, 0, 541, 523, 524, 504, 491, 536, 521, 502, 11, 4, 0, 0, 0, 0, 0, 0, 533, 506, 501, 499, 534, 489, 506, 474, 0, 0, 0, 0, 0, 0, 0, 0, 527, 517, 548, 502, 515, 480, 490, 548, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 517, 559, 531, 531, 509, 540, 0, 0, 0, 0, 0, 0, 0, 0, 482, 496, 506, 527, 508, 501, 511, 488, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4066, 4099, 4157, 4042, 4127, 4218, 4041] avg=4096.0000 max=4218 max_rank=6 min=4018 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16428 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 493, 479, 519, 548, 493, 511, 471, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 520, 519, 483, 523, 500, 517, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 515, 500, 565, 513, 504, 504, 5, 0, 0, 0, 0, 0, 0, 0, 490, 524, 513, 519, 486, 523, 498, 506, 10, 3, 0, 0, 0, 0, 0, 0, 520, 502, 529, 524, 510, 511, 478, 500, 0, 0, 0, 0, 0, 0, 0, 0, 481, 537, 509, 480, 514, 509, 516, 517, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 543, 525, 503, 504, 542, 497, 0, 0, 0, 0, 0, 0, 0, 0, 510, 533, 527, 530, 552, 537, 482, 483, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4092, 4139, 4072, 4074, 4063, 4117, 4175] avg=4096.0000 max=4175 max_rank=7 min=4036 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16429 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[496, 497, 515, 508, 501, 503, 502, 527, 0, 0, 0, 0, 0, 0, 0, 0, 555, 523, 485, 475, 518, 487, 555, 499, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 519, 522, 512, 499, 504, 496, 6, 1, 0, 0, 0, 0, 0, 0, 506, 527, 512, 515, 547, 511, 495, 537, 11, 4, 0, 0, 0, 0, 0, 0, 491, 529, 502, 505, 546, 523, 451, 554, 0, 0, 0, 0, 0, 0, 0, 0, 547, 504, 529, 507, 524, 480, 504, 540, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 498, 524, 494, 497, 554, 500, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 517, 485, 529, 494, 487, 486, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4097, 4089, 4165, 4101, 4135, 4105, 4027] avg=4096.0000 max=4165 max_rank=3 min=4027 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16368 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 520, 516, 524, 518, 494, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 486, 538, 551, 473, 496, 508, 532, 498, 0, 0, 0, 0, 0, 0, 0, 0, 485, 501, 524, 508, 485, 489, 519, 497, 6, 1, 0, 0, 0, 0, 0, 0, 518, 496, 521, 474, 501, 509, 529, 538, 11, 4, 0, 0, 0, 0, 0, 0, 545, 506, 511, 494, 536, 510, 499, 555, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 520, 497, 498, 516, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 534, 493, 455, 515, 553, 515, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 527, 542, 522, 490, 523, 475, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4082, 4015, 4101, 4156, 4070, 4076, 4121] avg=4096.0000 max=4156 max_rank=4 min=4015 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16423 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 523, 522, 520, 468, 529, 557, 539, 0, 0, 0, 0, 0, 0, 0, 0, 525, 478, 539, 500, 511, 487, 549, 511, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 508, 519, 506, 469, 517, 528, 6, 1, 0, 0, 0, 0, 0, 0, 542, 486, 487, 505, 509, 522, 490, 506, 11, 4, 0, 0, 0, 0, 0, 0, 499, 542, 518, 505, 527, 529, 482, 533, 0, 0, 0, 0, 0, 0, 0, 0, 504, 480, 499, 491, 526, 483, 505, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 536, 507, 504, 498, 531, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 516, 478, 519, 519, 518, 525, 531, 519, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4100, 4074, 4062, 4135, 3988, 4093, 4147] avg=4096.0000 max=4169 max_rank=0 min=3988 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16363 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 525, 490, 509, 528, 542, 526, 526, 0, 0, 0, 0, 0, 0, 0, 0, 517, 488, 540, 515, 503, 537, 501, 526, 0, 0, 0, 0, 0, 0, 0, 0, 472, 507, 487, 543, 524, 505, 533, 495, 9, 4, 3, 0, 0, 0, 0, 0, 478, 477, 464, 515, 504, 535, 519, 500, 15, 6, 0, 0, 0, 0, 0, 0, 536, 520, 497, 505, 486, 517, 497, 523, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 521, 483, 551, 532, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 542, 491, 485, 523, 507, 468, 539, 496, 0, 0, 0, 0, 0, 0, 0, 0, 478, 548, 484, 533, 483, 483, 562, 513, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4127, 4082, 4013, 4081, 4146, 4051, 4109] avg=4096.0000 max=4159 max_rank=0 min=4013 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 520, 523, 501, 487, 444, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 535, 518, 504, 490, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 513, 515, 540, 464, 522, 559, 6, 0, 0, 0, 0, 0, 0, 0, 469, 518, 473, 549, 539, 500, 473, 520, 11, 4, 0, 0, 0, 0, 0, 0, 487, 528, 530, 503, 538, 509, 521, 482, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 514, 507, 568, 546, 493, 530, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 505, 519, 534, 484, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 459, 507, 518, 498, 528, 510, 525, 508, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4067, 4144, 4056, 4098, 4169, 4095, 4076] avg=4096.0000 max=4169 max_rank=5 min=4056 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16438 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[509, 533, 525, 515, 530, 501, 538, 495, 524, 520, 485, 515, 523, 527, 490, 494, 524, 496, 500, 494, 522, 508, 506, 503, 510, 521, 485, 496, 514, 544, 513, 518, 501, 525, 499, 525, 528, 489, 515, 503, 513, 506, 487, 516, 510, 500, 551, 534, 492, 517, 536, 512, 512, 503, 552, 518, 515, 478, 524, 536, 478, 483, 489, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[516, 497, 494, 535, 532, 507, 505, 474, 484, 527, 509, 522, 491, 518, 445, 517, 557, 535, 530, 520, 535, 542, 517, 504, 535, 489, 514, 526, 518, 511, 538, 515, 529, 567, 548, 493, 520, 496, 507, 495, 509, 508, 523, 508, 541, 483, 522, 491, 487, 533, 496, 517, 513, 494, 486, 510, 518, 540, 492, 519, 514, 510, 454, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[492, 475, 534, 494, 503, 521, 530, 494, 538, 541, 481, 507, 490, 475, 493, 503, 499, 492, 506, 517, 540, 494, 499, 551, 504, 483, 523, 539, 524, 510, 537, 555, 499, 510, 479, 501, 505, 478, 532, 555, 514, 506, 500, 535, 499, 453, 494, 514, 501, 534, 507, 490, 543, 532, 525, 536, 464, 536, 501, 544, 566, 548, 511, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[507, 501, 495, 488, 514, 519, 503, 546, 479, 514, 495, 495, 497, 516, 487, 500, 486, 503, 501, 540, 528, 535, 533, 521, 512, 498, 533, 500, 509, 515, 538, 495, 525, 525, 517, 532, 551, 520, 496, 518, 503, 503, 493, 522, 550, 518, 484, 506, 544, 458, 524, 527, 493, 520, 481, 491, 530, 513, 495, 532, 514, 520, 510, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[499, 541, 478, 485, 505, 527, 482, 477, 509, 511, 509, 497, 530, 528, 513, 523, 453, 500, 512, 560, 532, 474, 538, 483, 522, 528, 482, 528, 523, 526, 451, 500, 545, 485, 504, 535, 517, 489, 524, 500, 514, 563, 540, 550, 495, 491, 524, 507, 536, 523, 523, 525, 477, 528, 514, 514, 532, 504, 461, 499, 530, 545, 523, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[510, 490, 557, 540, 526, 488, 493, 479, 517, 513, 483, 483, 515, 520, 491, 516, 538, 497, 517, 490, 503, 529, 546, 510, 519, 491, 546, 518, 539, 524, 511, 502, 530, 509, 486, 530, 511, 497, 516, 507, 515, 472, 559, 527, 495, 508, 509, 489, 476, 531, 488, 502, 467, 505, 543, 526, 552, 463, 541, 498, 523, 535, 489, 568] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[513, 513, 495, 496, 562, 527, 513, 565, 496, 524, 504, 483, 503, 504, 462, 507, 555, 518, 500, 485, 492, 485, 516, 539, 479, 471, 538, 532, 542, 504, 511, 502, 493, 502, 505, 555, 513, 532, 550, 509, 522, 485, 529, 506, 486, 520, 533, 500, 509, 546, 515, 516, 518, 513, 522, 560, 519, 500, 502, 477, 473, 510, 485, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[530, 532, 533, 517, 514, 534, 470, 476, 533, 490, 510, 481, 474, 532, 514, 522, 522, 509, 510, 486, 487, 506, 526, 545, 512, 501, 523, 540, 485, 503, 526, 511, 497, 523, 501, 490, 514, 491, 496, 524, 496, 509, 506, 513, 493, 558, 506, 478, 532, 506, 544, 501, 525, 514, 524, 517, 535, 553, 523, 516, 470, 514, 531, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[492, 475, 534, 494, 503, 521, 530, 494, 7, 2, 0, 0, 0, 0, 0, 0, 538, 541, 481, 507, 490, 475, 493, 503, 20, 18, 15, 4, 0, 0, 0, 0, 499, 492, 506, 517, 540, 494, 484, 551, 0, 0, 0, 0, 0, 0, 0, 0, 504, 483, 523, 521, 524, 510, 537, 555, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 479, 490, 505, 478, 532, 555, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 500, 535, 499, 453, 494, 514, 8, 0, 0, 0, 0, 0, 0, 0, 501, 534, 507, 490, 543, 532, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 464, 536, 501, 544, 566, 548, 511, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4085, 4083, 4157, 4048, 4023, 4148, 4172] avg=4096.0000 max=4172 max_rank=7 min=4023 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 497, 494, 535, 532, 507, 505, 474, 6, 2, 0, 0, 0, 0, 0, 0, 484, 527, 509, 522, 491, 518, 445, 517, 19, 18, 16, 4, 0, 0, 0, 0, 557, 535, 530, 520, 535, 542, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 535, 489, 514, 508, 518, 511, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 529, 567, 548, 483, 520, 496, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 523, 508, 541, 483, 522, 491, 8, 0, 0, 0, 0, 0, 0, 0, 487, 533, 496, 517, 513, 494, 486, 491, 0, 0, 0, 0, 0, 0, 0, 0, 518, 540, 492, 519, 514, 510, 454, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4070, 4224, 4128, 4145, 4093, 4017, 4023] avg=4096.0000 max=4224 max_rank=2 min=4017 min_rank=6 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 533, 525, 515, 530, 501, 538, 495, 7, 2, 0, 0, 0, 0, 0, 0, 524, 520, 485, 515, 523, 527, 490, 494, 19, 17, 16, 4, 0, 0, 0, 0, 524, 496, 500, 494, 522, 508, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 485, 479, 514, 544, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 501, 525, 499, 514, 528, 489, 515, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 487, 516, 510, 500, 551, 534, 9, 0, 0, 0, 0, 0, 0, 0, 492, 517, 536, 512, 512, 503, 552, 499, 0, 0, 0, 0, 0, 0, 0, 0, 515, 478, 524, 536, 478, 483, 489, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4134, 4037, 4084, 4074, 4126, 4123, 4035] avg=4096.0000 max=4155 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[507, 501, 495, 488, 514, 519, 503, 546, 7, 2, 0, 0, 0, 0, 0, 0, 479, 514, 495, 495, 497, 516, 487, 500, 18, 17, 16, 4, 0, 0, 0, 0, 486, 503, 501, 540, 528, 535, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 498, 533, 483, 509, 515, 538, 495, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 517, 521, 551, 520, 496, 518, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 493, 522, 550, 518, 484, 506, 9, 0, 0, 0, 0, 0, 0, 0, 544, 458, 524, 527, 493, 520, 481, 473, 0, 0, 0, 0, 0, 0, 0, 0, 530, 513, 495, 532, 514, 520, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4038, 4131, 4083, 4173, 4088, 4020, 4153] avg=4096.0000 max=4173 max_rank=4 min=4020 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 541, 478, 485, 505, 527, 482, 477, 7, 2, 0, 0, 0, 0, 0, 0, 509, 511, 509, 497, 530, 528, 513, 523, 19, 18, 17, 4, 0, 0, 0, 0, 453, 500, 512, 560, 532, 474, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 482, 510, 523, 526, 451, 500, 0, 0, 0, 0, 0, 0, 0, 0, 545, 485, 504, 524, 517, 489, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 514, 563, 540, 550, 495, 491, 524, 507, 9, 0, 0, 0, 0, 0, 0, 0, 536, 523, 523, 525, 477, 528, 514, 495, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 461, 499, 530, 545, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4178, 4035, 4042, 4088, 4193, 4121, 4108] avg=4096.0000 max=4193 max_rank=5 min=4003 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 490, 557, 540, 526, 488, 493, 479, 7, 2, 0, 0, 0, 0, 0, 0, 517, 513, 483, 483, 515, 520, 491, 516, 19, 17, 17, 4, 0, 0, 0, 0, 538, 497, 517, 490, 503, 529, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 491, 546, 501, 539, 524, 511, 502, 0, 0, 0, 0, 0, 0, 0, 0, 530, 509, 486, 519, 511, 497, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 472, 559, 527, 495, 508, 509, 489, 9, 0, 0, 0, 0, 0, 0, 0, 476, 531, 488, 502, 467, 505, 543, 507, 0, 0, 0, 0, 0, 0, 0, 0, 552, 463, 541, 498, 523, 535, 489, 557, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4095, 4113, 4133, 4075, 4083, 4019, 4158] avg=4096.0000 max=4158 max_rank=7 min=4019 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 513, 495, 496, 562, 527, 513, 565, 12, 6, 0, 0, 0, 0, 0, 0, 496, 524, 504, 483, 503, 504, 462, 507, 23, 21, 20, 8, 0, 0, 0, 0, 555, 518, 500, 485, 492, 485, 496, 539, 0, 0, 0, 0, 0, 0, 0, 0, 479, 471, 538, 511, 542, 504, 511, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 505, 535, 513, 532, 550, 509, 0, 0, 0, 0, 0, 0, 0, 0, 522, 485, 529, 506, 486, 520, 533, 500, 13, 0, 0, 0, 0, 0, 0, 0, 509, 546, 515, 516, 518, 513, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 502, 477, 473, 510, 485, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4055, 4070, 4058, 4139, 4094, 4176, 3974] avg=4096.0000 max=4202 max_rank=0 min=3974 min_rank=7 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 532, 533, 517, 514, 534, 470, 476, 6, 2, 0, 0, 0, 0, 0, 0, 533, 490, 510, 481, 474, 532, 514, 522, 19, 18, 16, 4, 0, 0, 0, 0, 522, 509, 510, 486, 487, 506, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 523, 522, 485, 503, 526, 511, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 501, 480, 514, 491, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 496, 509, 506, 513, 493, 558, 506, 478, 9, 0, 0, 0, 0, 0, 0, 0, 532, 506, 544, 501, 525, 514, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 553, 523, 516, 470, 514, 531, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4113, 4075, 4083, 4026, 4068, 4144, 4145] avg=4096.0000 max=4145 max_rank=7 min=4026 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 256, 256, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[539, 513, 507, 509, 502, 531, 496, 506, 512, 518, 501, 492, 504, 501, 500, 494, 512, 529, 478, 521, 553, 484, 515, 502, 491, 539, 482, 494, 544, 483, 540, 531, 530, 512, 520, 498, 502, 535, 542, 534, 524, 512, 536, 484, 507, 523, 502, 463, 520, 531, 498, 518, 529, 511, 533, 526, 483, 512, 515, 505, 497, 519, 512, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[463, 490, 504, 490, 544, 518, 552, 521, 537, 507, 487, 501, 528, 538, 482, 541, 537, 514, 496, 531, 481, 517, 485, 525, 531, 519, 506, 514, 512, 475, 520, 489, 510, 491, 523, 496, 541, 522, 501, 513, 510, 498, 538, 506, 480, 522, 486, 531, 487, 529, 532, 490, 516, 539, 488, 513, 510, 506, 530, 528, 500, 512, 519, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[522, 502, 507, 476, 515, 487, 515, 550, 514, 511, 499, 530, 494, 536, 502, 516, 487, 491, 534, 526, 455, 527, 515, 511, 472, 503, 532, 508, 522, 548, 521, 506, 510, 531, 569, 517, 518, 500, 482, 524, 467, 553, 520, 512, 479, 557, 540, 536, 493, 481, 535, 472, 495, 468, 515, 510, 491, 502, 492, 526, 536, 529, 539, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[492, 528, 517, 462, 512, 549, 520, 488, 508, 542, 504, 508, 500, 499, 487, 531, 550, 497, 493, 516, 549, 511, 524, 528, 484, 527, 494, 486, 511, 500, 493, 536, 505, 497, 518, 503, 539, 480, 523, 522, 512, 514, 503, 493, 477, 512, 493, 489, 489, 505, 529, 530, 499, 482, 498, 535, 493, 536, 524, 556, 556, 571, 520, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[456, 523, 519, 515, 520, 544, 513, 490, 504, 531, 513, 521, 518, 513, 549, 510, 533, 511, 489, 492, 512, 494, 482, 514, 488, 544, 518, 519, 529, 532, 522, 505, 520, 553, 529, 534, 515, 492, 527, 488, 494, 504, 487, 490, 536, 517, 498, 455, 545, 507, 483, 482, 539, 480, 536, 508, 497, 551, 492, 548, 540, 464, 525, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[525, 525, 516, 505, 509, 506, 532, 577, 538, 495, 526, 473, 522, 503, 511, 520, 535, 499, 505, 538, 525, 488, 495, 515, 503, 503, 545, 495, 512, 500, 477, 504, 501, 496, 515, 500, 516, 491, 477, 497, 504, 452, 500, 526, 515, 516, 534, 500, 513, 529, 507, 556, 517, 495, 524, 556, 521, 534, 504, 503, 500, 513, 537, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[546, 463, 525, 513, 528, 516, 542, 522, 495, 477, 510, 485, 498, 497, 511, 500, 532, 515, 518, 499, 462, 514, 550, 503, 497, 510, 529, 497, 511, 536, 537, 494, 523, 510, 502, 499, 518, 503, 518, 501, 516, 487, 550, 506, 532, 544, 550, 542, 510, 514, 560, 531, 508, 514, 505, 508, 497, 483, 519, 480, 498, 482, 528, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[513, 532, 521, 527, 555, 513, 502, 503, 522, 490, 521, 497, 508, 517, 494, 519, 537, 533, 541, 511, 495, 513, 534, 504, 493, 504, 518, 515, 528, 475, 523, 533, 509, 422, 500, 502, 511, 491, 555, 516, 497, 562, 489, 529, 534, 499, 531, 530, 491, 469, 522, 545, 535, 499, 499, 492, 508, 497, 522, 482, 493, 526, 513, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[539, 513, 507, 509, 484, 531, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 501, 492, 504, 501, 500, 494, 18, 0, 0, 0, 0, 0, 0, 0, 511, 529, 478, 521, 553, 484, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 491, 539, 482, 494, 544, 483, 540, 531, 7, 0, 0, 0, 0, 0, 0, 0, 530, 512, 520, 498, 502, 535, 542, 534, 1, 0, 0, 0, 0, 0, 0, 0, 524, 512, 536, 484, 507, 523, 502, 463, 7, 0, 0, 0, 0, 0, 0, 0, 520, 531, 498, 518, 529, 511, 533, 526, 0, 0, 0, 0, 0, 0, 0, 0, 483, 512, 515, 505, 497, 519, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4040, 4093, 4111, 4174, 4058, 4166, 4041] avg=4096.0000 max=4174 max_rank=4 min=4040 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 528, 517, 462, 494, 549, 520, 488, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 504, 508, 500, 499, 487, 531, 18, 0, 0, 0, 0, 0, 0, 0, 549, 497, 493, 516, 549, 511, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 494, 486, 511, 500, 493, 536, 7, 0, 0, 0, 0, 0, 0, 0, 505, 497, 518, 503, 539, 480, 523, 522, 1, 0, 0, 0, 0, 0, 0, 0, 512, 514, 503, 493, 477, 512, 493, 489, 7, 0, 0, 0, 0, 0, 0, 0, 489, 505, 529, 530, 499, 482, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 493, 536, 524, 556, 556, 571, 506, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4097, 4167, 4038, 4088, 4000, 4067, 4261] avg=4096.0000 max=4261 max_rank=7 min=4000 min_rank=5 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[463, 490, 504, 490, 524, 518, 552, 521, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 487, 501, 528, 538, 482, 541, 20, 0, 0, 0, 0, 0, 0, 0, 536, 514, 496, 531, 481, 517, 485, 525, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 506, 514, 512, 475, 520, 489, 7, 0, 0, 0, 0, 0, 0, 0, 510, 491, 523, 496, 541, 522, 501, 513, 1, 0, 0, 0, 0, 0, 0, 0, 510, 498, 538, 506, 480, 522, 486, 531, 7, 0, 0, 0, 0, 0, 0, 0, 487, 529, 532, 490, 516, 539, 488, 513, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 530, 528, 500, 512, 505, 546, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4141, 4085, 4073, 4098, 4078, 4094, 4137] avg=4096.0000 max=4141 max_rank=1 min=4062 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 507, 476, 496, 487, 515, 550, 0, 0, 0, 0, 0, 0, 0, 0, 514, 511, 499, 530, 494, 536, 502, 516, 19, 0, 0, 0, 0, 0, 0, 0, 486, 491, 534, 526, 455, 527, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 472, 503, 532, 508, 522, 548, 521, 506, 7, 0, 0, 0, 0, 0, 0, 0, 510, 531, 569, 517, 518, 500, 482, 524, 1, 1, 0, 0, 0, 0, 0, 0, 467, 553, 520, 512, 479, 557, 540, 536, 8, 0, 0, 0, 0, 0, 0, 0, 493, 481, 535, 472, 495, 468, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 492, 526, 536, 529, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4121, 4045, 4119, 4153, 4172, 3969, 4134] avg=4096.0000 max=4172 max_rank=5 min=3969 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[456, 523, 519, 515, 501, 544, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 504, 531, 513, 521, 518, 513, 549, 510, 19, 0, 0, 0, 0, 0, 0, 0, 532, 511, 489, 492, 512, 494, 482, 514, 0, 0, 0, 0, 0, 0, 0, 0, 488, 544, 518, 519, 529, 532, 522, 505, 7, 0, 0, 0, 0, 0, 0, 0, 520, 553, 529, 534, 515, 492, 527, 488, 1, 1, 0, 0, 0, 0, 0, 0, 494, 504, 487, 490, 536, 517, 498, 455, 7, 0, 0, 0, 0, 0, 0, 0, 545, 507, 483, 482, 539, 480, 536, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 551, 492, 548, 540, 464, 510, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4178, 4026, 4164, 4160, 3988, 4080, 4111] avg=4096.0000 max=4178 max_rank=1 min=3988 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 525, 516, 505, 487, 506, 532, 577, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 526, 473, 522, 503, 511, 520, 22, 1, 0, 0, 0, 0, 0, 0, 528, 499, 505, 538, 525, 488, 495, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 545, 495, 512, 500, 477, 504, 10, 0, 0, 0, 0, 0, 0, 0, 501, 496, 515, 500, 516, 491, 477, 497, 7, 5, 7, 0, 0, 0, 0, 0, 504, 452, 500, 526, 515, 516, 534, 500, 13, 0, 0, 0, 0, 0, 0, 0, 513, 529, 500, 556, 517, 495, 524, 556, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 504, 503, 500, 513, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4111, 4093, 4049, 4012, 4060, 4190, 4080] avg=4096.0000 max=4190 max_rank=6 min=4012 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[546, 463, 525, 513, 509, 516, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 477, 510, 485, 498, 497, 511, 500, 19, 0, 0, 0, 0, 0, 0, 0, 531, 515, 518, 499, 462, 514, 550, 503, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 529, 497, 511, 536, 537, 494, 7, 0, 0, 0, 0, 0, 0, 0, 523, 510, 502, 499, 518, 503, 518, 501, 1, 1, 0, 0, 0, 0, 0, 0, 516, 487, 550, 506, 532, 544, 550, 542, 7, 0, 0, 0, 0, 0, 0, 0, 510, 514, 560, 531, 508, 514, 505, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 519, 480, 498, 482, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 3992, 4092, 4118, 4076, 4234, 4150, 3970] avg=4096.0000 max=4234 max_rank=5 min=3970 min_rank=7 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 532, 521, 527, 535, 513, 502, 503, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 521, 497, 508, 517, 494, 519, 20, 0, 0, 0, 0, 0, 0, 0, 536, 533, 541, 511, 495, 513, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 493, 504, 518, 515, 528, 475, 523, 533, 7, 0, 0, 0, 0, 0, 0, 0, 509, 422, 500, 502, 511, 491, 555, 516, 1, 0, 0, 0, 0, 0, 0, 0, 497, 562, 489, 529, 534, 499, 531, 530, 7, 0, 0, 0, 0, 0, 0, 0, 491, 469, 522, 545, 535, 499, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 522, 482, 493, 526, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4088, 4167, 4096, 4007, 4178, 4052, 4034] avg=4096.0000 max=4178 max_rank=5 min=4007 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[514, 522, 529, 501, 494, 541, 488, 510, 534, 507, 494, 521, 501, 514, 493, 510, 483, 543, 530, 489, 517, 509, 529, 476, 517, 536, 504, 497, 517, 502, 560, 514, 538, 483, 488, 481, 477, 515, 549, 499, 522, 545, 528, 513, 516, 510, 508, 506, 449, 527, 503, 524, 513, 508, 507, 557, 498, 531, 519, 530, 508, 525, 484, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[496, 512, 516, 531, 476, 486, 523, 483, 485, 503, 478, 538, 505, 515, 475, 524, 515, 514, 513, 529, 573, 484, 529, 520, 510, 513, 504, 540, 511, 500, 488, 496, 521, 500, 506, 524, 539, 525, 509, 502, 500, 464, 503, 521, 546, 510, 543, 532, 502, 526, 529, 513, 501, 520, 555, 495, 520, 488, 520, 531, 523, 482, 520, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[509, 533, 488, 493, 528, 492, 510, 524, 544, 529, 515, 513, 501, 520, 487, 546, 547, 483, 537, 494, 502, 529, 510, 457, 520, 510, 467, 493, 537, 515, 489, 558, 497, 516, 513, 522, 542, 531, 505, 491, 516, 499, 528, 471, 515, 507, 488, 514, 503, 516, 548, 549, 516, 500, 497, 531, 544, 498, 519, 509, 490, 514, 488, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[526, 529, 511, 491, 520, 557, 503, 533, 516, 513, 529, 507, 518, 493, 514, 480, 496, 541, 529, 528, 475, 484, 516, 535, 533, 486, 505, 524, 528, 557, 529, 515, 517, 498, 527, 527, 515, 519, 501, 491, 490, 513, 548, 467, 515, 523, 460, 523, 508, 533, 514, 487, 474, 496, 455, 502, 524, 498, 503, 528, 509, 527, 510, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[511, 525, 500, 568, 464, 507, 530, 538, 517, 522, 505, 513, 474, 504, 499, 519, 485, 505, 545, 536, 502, 478, 494, 502, 502, 514, 478, 510, 490, 560, 525, 540, 524, 533, 500, 524, 537, 477, 523, 534, 535, 512, 527, 513, 487, 493, 504, 505, 485, 506, 516, 511, 517, 472, 520, 528, 509, 465, 515, 529, 520, 524, 528, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[481, 529, 546, 481, 521, 527, 526, 534, 533, 506, 494, 530, 548, 538, 525, 516, 493, 485, 501, 511, 471, 510, 511, 518, 487, 540, 513, 513, 509, 499, 490, 540, 523, 508, 517, 514, 516, 542, 497, 541, 529, 516, 469, 488, 531, 491, 500, 517, 539, 512, 474, 528, 481, 516, 484, 484, 520, 500, 546, 503, 551, 523, 496, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[511, 516, 510, 536, 530, 535, 497, 534, 504, 492, 514, 481, 472, 496, 523, 509, 521, 504, 511, 511, 532, 488, 502, 520, 539, 531, 483, 492, 513, 493, 534, 500, 463, 486, 514, 503, 533, 527, 505, 485, 521, 472, 494, 496, 515, 544, 526, 498, 489, 517, 479, 516, 530, 536, 521, 536, 534, 560, 493, 529, 548, 499, 518, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[512, 525, 492, 540, 520, 524, 464, 478, 519, 508, 517, 506, 536, 549, 525, 548, 502, 512, 486, 501, 496, 516, 502, 500, 483, 539, 463, 510, 538, 502, 523, 520, 503, 495, 467, 507, 532, 529, 531, 539, 503, 536, 525, 514, 538, 528, 483, 544, 485, 513, 518, 532, 509, 450, 534, 457, 520, 497, 502, 533, 498, 519, 548, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 519, 488, 493, 528, 492, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 544, 529, 515, 513, 501, 520, 487, 546, 0, 0, 0, 0, 0, 0, 0, 0, 547, 483, 537, 494, 502, 529, 510, 457, 14, 6, 4, 0, 0, 0, 0, 0, 520, 510, 467, 493, 537, 515, 489, 548, 0, 0, 0, 0, 0, 0, 0, 0, 497, 516, 513, 522, 539, 531, 505, 491, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 528, 471, 515, 507, 488, 514, 6, 3, 0, 0, 0, 0, 0, 0, 503, 516, 548, 549, 516, 500, 497, 531, 26, 0, 0, 0, 0, 0, 0, 0, 544, 498, 519, 477, 490, 514, 488, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4155, 4083, 4079, 4114, 4047, 4186, 4041] avg=4096.0000 max=4186 max_rank=6 min=4041 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 529, 501, 494, 541, 488, 510, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 494, 521, 501, 514, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 483, 543, 530, 489, 517, 509, 529, 476, 14, 6, 4, 0, 0, 0, 0, 0, 517, 536, 504, 497, 517, 502, 560, 505, 0, 0, 0, 0, 0, 0, 0, 0, 538, 483, 488, 481, 475, 515, 549, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 545, 528, 513, 516, 510, 508, 506, 5, 2, 0, 0, 0, 0, 0, 0, 449, 527, 503, 524, 513, 508, 507, 557, 27, 0, 0, 0, 0, 0, 0, 0, 498, 531, 519, 497, 508, 525, 484, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4074, 4100, 4138, 4028, 4155, 4115, 4073] avg=4096.0000 max=4155 max_rank=5 min=4028 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 499, 516, 531, 476, 486, 523, 483, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 478, 538, 505, 515, 475, 524, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 513, 529, 573, 484, 529, 520, 13, 6, 4, 0, 0, 0, 0, 0, 510, 513, 504, 540, 511, 500, 488, 487, 0, 0, 0, 0, 0, 0, 0, 0, 521, 500, 506, 524, 536, 525, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 500, 464, 503, 521, 546, 510, 543, 532, 5, 3, 0, 0, 0, 0, 0, 0, 502, 526, 529, 513, 501, 520, 555, 495, 27, 0, 0, 0, 0, 0, 0, 0, 520, 488, 520, 498, 523, 482, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4023, 4200, 4053, 4123, 4127, 4168, 4064] avg=4096.0000 max=4200 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[526, 515, 511, 491, 520, 557, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 529, 507, 518, 493, 514, 480, 0, 0, 0, 0, 0, 0, 0, 0, 496, 541, 529, 528, 475, 484, 516, 535, 14, 6, 4, 0, 0, 0, 0, 0, 533, 486, 505, 524, 528, 557, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 527, 527, 512, 519, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 548, 467, 515, 523, 460, 523, 5, 3, 0, 0, 0, 0, 0, 0, 508, 533, 514, 487, 474, 496, 455, 502, 27, 0, 0, 0, 0, 0, 0, 0, 524, 498, 503, 495, 509, 527, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4070, 4128, 4168, 4092, 4047, 3996, 4111] avg=4096.0000 max=4168 max_rank=3 min=3996 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 511, 500, 568, 464, 507, 530, 538, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 505, 513, 474, 504, 499, 519, 0, 0, 0, 0, 0, 0, 0, 0, 485, 505, 545, 536, 502, 478, 494, 502, 14, 6, 4, 0, 0, 0, 0, 0, 502, 514, 478, 510, 490, 560, 525, 531, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 500, 524, 534, 477, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 535, 512, 527, 513, 487, 493, 504, 505, 5, 3, 0, 0, 0, 0, 0, 0, 485, 506, 516, 511, 517, 472, 520, 528, 27, 0, 0, 0, 0, 0, 0, 0, 509, 465, 515, 496, 520, 524, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4053, 4071, 4110, 4149, 4084, 4082, 4090] avg=4096.0000 max=4149 max_rank=4 min=4053 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[481, 515, 546, 481, 521, 527, 526, 534, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 494, 530, 548, 538, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 501, 511, 471, 510, 511, 518, 14, 6, 4, 0, 0, 0, 0, 0, 487, 540, 513, 513, 509, 499, 490, 531, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 517, 514, 513, 542, 497, 541, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 469, 488, 531, 491, 500, 517, 5, 3, 0, 0, 0, 0, 0, 0, 539, 512, 474, 528, 481, 516, 484, 484, 25, 0, 0, 0, 0, 0, 0, 0, 520, 500, 546, 472, 551, 523, 496, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4190, 4024, 4082, 4155, 4049, 4043, 4094] avg=4096.0000 max=4190 max_rank=1 min=4024 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 508, 492, 540, 520, 524, 464, 478, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 517, 506, 536, 549, 525, 548, 4, 0, 0, 0, 0, 0, 0, 0, 502, 512, 486, 501, 496, 516, 502, 500, 17, 11, 7, 0, 0, 0, 0, 0, 483, 539, 463, 510, 538, 502, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 467, 507, 523, 529, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 525, 514, 538, 528, 483, 544, 9, 5, 0, 0, 0, 0, 0, 0, 485, 513, 518, 532, 509, 450, 534, 457, 29, 0, 0, 0, 0, 0, 0, 0, 520, 497, 502, 493, 498, 519, 548, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4212, 4050, 4062, 4094, 4185, 4027, 4100] avg=4096.0000 max=4212 max_rank=1 min=4027 min_rank=6 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 503, 510, 536, 530, 535, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 504, 492, 514, 481, 472, 496, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 511, 511, 532, 488, 502, 520, 13, 6, 4, 0, 0, 0, 0, 0, 539, 531, 483, 492, 513, 493, 534, 491, 0, 0, 0, 0, 0, 0, 0, 0, 463, 486, 514, 503, 530, 527, 505, 485, 0, 0, 0, 0, 0, 0, 0, 0, 521, 472, 494, 496, 515, 544, 526, 498, 5, 3, 0, 0, 0, 0, 0, 0, 489, 517, 479, 516, 530, 536, 521, 536, 27, 0, 0, 0, 0, 0, 0, 0, 534, 560, 493, 496, 548, 499, 518, 547, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 3991, 4112, 4076, 4013, 4074, 4151, 4195] avg=4096.0000 max=4195 max_rank=7 min=3991 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[498, 542, 496, 519, 514, 509, 486, 499, 518, 484, 504, 510, 483, 534, 488, 494, 497, 532, 528, 543, 528, 529, 482, 471, 515, 517, 505, 477, 492, 511, 486, 546, 516, 527, 518, 511, 541, 519, 524, 530, 505, 493, 537, 519, 495, 502, 499, 532, 513, 539, 534, 514, 473, 499, 515, 524, 539, 476, 541, 507, 534, 549, 511, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[516, 538, 503, 560, 510, 532, 496, 536, 545, 505, 500, 523, 542, 519, 505, 486, 517, 495, 515, 483, 466, 510, 506, 512, 500, 488, 519, 536, 536, 540, 494, 484, 511, 490, 551, 523, 487, 519, 542, 482, 516, 502, 536, 519, 503, 534, 489, 562, 547, 474, 521, 489, 507, 498, 517, 499, 493, 480, 521, 485, 490, 516, 525, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[483, 524, 518, 487, 508, 506, 495, 507, 509, 508, 525, 540, 482, 556, 527, 498, 498, 492, 492, 503, 516, 505, 503, 455, 527, 529, 503, 527, 510, 542, 526, 527, 498, 516, 540, 517, 521, 489, 517, 507, 509, 491, 529, 503, 503, 505, 498, 522, 522, 489, 518, 505, 496, 507, 529, 522, 482, 533, 524, 528, 516, 548, 537, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[473, 501, 529, 499, 492, 510, 500, 517, 519, 481, 502, 558, 547, 543, 510, 506, 511, 485, 513, 536, 507, 541, 514, 512, 503, 527, 506, 542, 489, 503, 523, 509, 516, 487, 471, 494, 523, 530, 532, 518, 515, 559, 510, 498, 494, 539, 493, 490, 531, 534, 503, 537, 514, 487, 532, 538, 494, 509, 507, 495, 485, 538, 509, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[515, 556, 526, 512, 528, 511, 477, 487, 544, 511, 529, 555, 527, 544, 491, 523, 486, 489, 550, 514, 510, 500, 491, 493, 503, 536, 509, 516, 481, 523, 495, 501, 502, 559, 509, 516, 502, 553, 488, 474, 515, 525, 500, 508, 531, 488, 518, 468, 496, 493, 504, 490, 533, 486, 525, 530, 513, 564, 520, 499, 524, 480, 510, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[493, 499, 504, 508, 526, 508, 515, 492, 516, 509, 512, 547, 517, 519, 505, 497, 520, 523, 512, 527, 527, 513, 524, 514, 573, 548, 532, 534, 529, 513, 497, 510, 496, 515, 496, 504, 539, 486, 486, 501, 481, 531, 543, 498, 519, 483, 468, 519, 507, 492, 515, 492, 516, 497, 518, 544, 511, 517, 467, 557, 538, 501, 478, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[521, 527, 485, 539, 530, 511, 525, 513, 501, 535, 450, 545, 537, 503, 523, 511, 471, 526, 515, 488, 497, 510, 499, 507, 496, 520, 513, 507, 513, 506, 504, 525, 521, 537, 517, 505, 463, 523, 541, 522, 491, 530, 516, 539, 495, 524, 501, 544, 507, 474, 502, 583, 510, 514, 478, 481, 493, 500, 526, 492, 534, 542, 487, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[503, 505, 540, 475, 542, 531, 494, 498, 493, 509, 517, 518, 500, 502, 479, 514, 500, 548, 531, 500, 473, 470, 532, 534, 479, 515, 492, 521, 456, 494, 529, 496, 557, 513, 497, 562, 541, 481, 468, 537, 555, 533, 567, 552, 517, 504, 497, 522, 545, 501, 500, 500, 504, 500, 495, 547, 530, 515, 484, 512, 520, 500, 478, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[473, 501, 529, 499, 492, 510, 500, 517, 7, 0, 0, 0, 0, 0, 0, 0, 519, 481, 502, 537, 547, 543, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 511, 485, 513, 536, 507, 541, 514, 512, 21, 13, 0, 0, 0, 0, 0, 0, 503, 523, 506, 542, 489, 503, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 516, 475, 471, 494, 523, 530, 532, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 559, 497, 498, 494, 539, 493, 490, 0, 0, 0, 0, 0, 0, 0, 0, 531, 534, 503, 537, 514, 487, 532, 538, 5, 2, 0, 0, 0, 0, 0, 0, 494, 509, 507, 495, 485, 538, 509, 478, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4145, 4153, 4098, 4059, 4085, 4183, 4017] avg=4096.0000 max=4183 max_rank=6 min=4017 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 538, 503, 560, 510, 532, 496, 536, 7, 0, 0, 0, 0, 0, 0, 0, 545, 505, 500, 503, 542, 519, 505, 486, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 515, 483, 466, 510, 506, 512, 20, 14, 0, 0, 0, 0, 0, 0, 500, 485, 519, 536, 536, 540, 494, 484, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 551, 523, 487, 519, 542, 482, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 522, 519, 503, 534, 489, 562, 0, 0, 0, 0, 0, 0, 0, 0, 547, 474, 521, 489, 507, 498, 517, 499, 5, 1, 0, 0, 0, 0, 0, 0, 493, 480, 521, 485, 490, 516, 525, 523, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4105, 4038, 4094, 4093, 4147, 4058, 4035] avg=4096.0000 max=4198 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 542, 496, 519, 514, 509, 486, 499, 8, 0, 0, 0, 0, 0, 0, 0, 518, 484, 504, 491, 483, 534, 488, 494, 0, 0, 0, 0, 0, 0, 0, 0, 497, 532, 528, 543, 528, 529, 482, 471, 19, 14, 0, 0, 0, 0, 0, 0, 515, 513, 505, 477, 492, 511, 486, 546, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 518, 511, 541, 519, 524, 530, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 523, 519, 495, 502, 499, 532, 0, 0, 0, 0, 0, 0, 0, 0, 513, 539, 534, 514, 473, 499, 515, 524, 5, 2, 0, 0, 0, 0, 0, 0, 539, 476, 541, 507, 534, 549, 511, 495, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 3996, 4143, 4045, 4173, 4068, 4118, 4154] avg=4096.0000 max=4173 max_rank=4 min=3996 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 499, 504, 508, 526, 508, 515, 492, 8, 0, 0, 0, 0, 0, 0, 0, 516, 509, 512, 526, 517, 519, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 512, 527, 527, 513, 524, 514, 21, 14, 0, 0, 0, 0, 0, 0, 573, 544, 532, 534, 529, 513, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 496, 504, 539, 486, 486, 501, 0, 0, 0, 0, 0, 0, 0, 0, 481, 531, 529, 498, 519, 483, 468, 519, 0, 0, 0, 0, 0, 0, 0, 0, 507, 492, 515, 492, 516, 497, 518, 544, 5, 2, 0, 0, 0, 0, 0, 0, 511, 517, 467, 557, 538, 501, 478, 490, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4101, 4195, 4232, 4010, 4028, 4088, 4061] avg=4096.0000 max=4232 max_rank=3 min=4010 min_rank=4 imbalance_ratio=1.033203 max_load_reduction=28536 max_load_reduction_pct=0.870850 moved_to_echo=16187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 556, 526, 512, 528, 511, 477, 487, 8, 0, 0, 0, 0, 0, 0, 0, 544, 511, 529, 534, 527, 544, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 486, 489, 550, 514, 510, 500, 491, 493, 21, 13, 0, 0, 0, 0, 0, 0, 503, 532, 509, 516, 481, 523, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 545, 509, 516, 502, 553, 488, 474, 0, 0, 0, 0, 0, 0, 0, 0, 515, 525, 487, 508, 531, 488, 518, 468, 0, 0, 0, 0, 0, 0, 0, 0, 496, 493, 504, 490, 533, 486, 525, 530, 6, 2, 0, 0, 0, 0, 0, 0, 513, 564, 520, 499, 524, 480, 510, 512, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4203, 4067, 4060, 4089, 4040, 4065, 4124] avg=4096.0000 max=4203 max_rank=1 min=4040 min_rank=5 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[483, 524, 518, 487, 508, 506, 495, 507, 8, 0, 0, 0, 0, 0, 0, 0, 509, 508, 525, 520, 482, 556, 527, 498, 0, 0, 0, 0, 0, 0, 0, 0, 498, 492, 492, 503, 516, 505, 503, 455, 20, 13, 0, 0, 0, 0, 0, 0, 527, 525, 503, 527, 510, 542, 526, 527, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 540, 517, 521, 489, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 516, 503, 503, 505, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 518, 505, 496, 507, 529, 522, 5, 2, 0, 0, 0, 0, 0, 0, 482, 533, 524, 528, 516, 548, 537, 519, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4125, 3997, 4187, 4092, 4047, 4095, 4189] avg=4096.0000 max=4189 max_rank=7 min=3997 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 527, 485, 539, 530, 511, 525, 513, 11, 4, 0, 0, 0, 0, 0, 0, 501, 535, 450, 521, 537, 503, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 471, 526, 515, 488, 497, 510, 499, 507, 24, 16, 0, 0, 0, 0, 0, 0, 496, 510, 513, 507, 513, 506, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 517, 505, 463, 523, 541, 522, 0, 0, 0, 0, 0, 0, 0, 0, 491, 530, 496, 539, 495, 524, 501, 544, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 502, 583, 510, 514, 478, 481, 9, 4, 0, 0, 0, 0, 0, 0, 493, 500, 526, 492, 534, 542, 487, 523, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4081, 4053, 4074, 4109, 4120, 4062, 4103] avg=4096.0000 max=4166 max_rank=0 min=4053 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 505, 540, 475, 542, 531, 494, 498, 8, 0, 0, 0, 0, 0, 0, 0, 493, 509, 517, 498, 500, 502, 479, 514, 0, 0, 0, 0, 0, 0, 0, 0, 500, 548, 531, 500, 473, 470, 532, 534, 20, 14, 0, 0, 0, 0, 0, 0, 479, 511, 492, 521, 456, 494, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0, 557, 500, 497, 562, 541, 481, 468, 537, 0, 0, 0, 0, 0, 0, 0, 0, 555, 533, 553, 552, 517, 504, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 545, 501, 500, 500, 504, 500, 495, 547, 5, 2, 0, 0, 0, 0, 0, 0, 530, 515, 484, 512, 520, 500, 478, 544, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4012, 4122, 3978, 4143, 4233, 4099, 4085] avg=4096.0000 max=4233 max_rank=5 min=3978 min_rank=3 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[536, 492, 522, 522, 489, 523, 509, 438, 526, 481, 506, 531, 492, 536, 533, 543, 495, 502, 479, 521, 529, 501, 498, 474, 522, 528, 439, 524, 478, 489, 549, 489, 520, 517, 522, 499, 499, 502, 531, 489, 531, 545, 525, 540, 546, 538, 462, 497, 536, 527, 522, 517, 507, 503, 494, 564, 498, 527, 539, 506, 530, 488, 526, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[566, 488, 504, 490, 537, 502, 499, 511, 515, 483, 535, 524, 498, 561, 467, 486, 500, 481, 542, 529, 493, 462, 515, 492, 506, 517, 544, 532, 498, 482, 493, 501, 558, 566, 528, 517, 505, 514, 523, 515, 515, 531, 490, 554, 533, 502, 506, 511, 551, 497, 504, 505, 475, 540, 504, 514, 524, 519, 539, 522, 507, 483, 486, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[532, 503, 540, 532, 444, 520, 510, 487, 513, 494, 499, 495, 491, 480, 513, 515, 521, 525, 448, 533, 499, 515, 519, 479, 531, 549, 509, 489, 520, 526, 497, 532, 527, 562, 497, 532, 500, 511, 546, 511, 509, 502, 528, 541, 485, 501, 499, 502, 534, 508, 508, 520, 543, 439, 530, 498, 535, 536, 520, 540, 501, 487, 529, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[514, 509, 481, 502, 485, 486, 501, 528, 492, 525, 490, 516, 500, 515, 543, 500, 505, 484, 468, 516, 481, 519, 504, 519, 550, 559, 505, 472, 520, 504, 498, 502, 529, 531, 529, 540, 519, 488, 500, 566, 517, 500, 515, 534, 526, 541, 501, 513, 515, 522, 539, 521, 468, 467, 464, 521, 527, 503, 528, 553, 525, 535, 504, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[520, 529, 543, 492, 520, 484, 511, 505, 485, 544, 517, 507, 512, 512, 517, 518, 498, 497, 492, 479, 542, 502, 530, 518, 562, 530, 554, 457, 518, 544, 519, 538, 477, 535, 497, 537, 525, 517, 511, 474, 516, 495, 487, 500, 502, 510, 488, 528, 512, 544, 494, 475, 511, 535, 473, 499, 488, 520, 513, 509, 553, 510, 536, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[523, 500, 505, 498, 499, 515, 518, 510, 502, 480, 521, 483, 511, 467, 528, 500, 492, 523, 513, 504, 526, 506, 530, 491, 518, 509, 540, 504, 534, 517, 511, 523, 528, 502, 524, 519, 522, 505, 502, 498, 566, 530, 510, 512, 491, 509, 487, 530, 561, 518, 484, 509, 506, 531, 529, 547, 495, 508, 522, 496, 522, 494, 512, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[477, 523, 487, 482, 549, 513, 508, 509, 506, 489, 513, 518, 475, 521, 552, 510, 493, 548, 520, 479, 568, 520, 505, 517, 543, 518, 479, 487, 480, 494, 502, 536, 496, 499, 496, 513, 556, 515, 553, 524, 481, 504, 520, 547, 516, 519, 502, 519, 543, 553, 504, 504, 517, 485, 534, 486, 495, 548, 530, 512, 474, 500, 502, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[486, 514, 460, 518, 488, 532, 527, 510, 509, 488, 538, 501, 463, 486, 544, 504, 520, 492, 561, 528, 510, 531, 519, 500, 510, 529, 535, 508, 535, 511, 523, 486, 546, 484, 529, 530, 495, 504, 541, 523, 491, 501, 507, 498, 517, 489, 482, 518, 533, 504, 502, 512, 499, 522, 484, 530, 535, 531, 492, 531, 521, 522, 525, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[536, 492, 522, 522, 489, 523, 509, 438, 39, 0, 0, 0, 0, 0, 0, 0, 526, 481, 506, 531, 492, 536, 533, 543, 17, 5, 4, 3, 0, 0, 0, 0, 495, 502, 479, 521, 529, 501, 498, 474, 22, 11, 0, 0, 0, 0, 0, 0, 505, 528, 439, 524, 478, 489, 549, 489, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 522, 499, 499, 502, 481, 489, 0, 0, 0, 0, 0, 0, 0, 0, 531, 545, 525, 535, 546, 538, 462, 497, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 522, 517, 507, 503, 494, 564, 0, 0, 0, 0, 0, 0, 0, 0, 498, 501, 539, 506, 530, 488, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4177, 4032, 4001, 4029, 4179, 4167, 4113] avg=4096.0000 max=4179 max_rank=5 min=4001 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[532, 503, 540, 532, 444, 520, 510, 487, 40, 0, 0, 0, 0, 0, 0, 0, 513, 494, 499, 495, 491, 480, 513, 515, 17, 5, 4, 3, 0, 0, 0, 0, 521, 525, 448, 533, 499, 515, 519, 479, 22, 11, 0, 0, 0, 0, 0, 0, 514, 549, 509, 489, 520, 526, 497, 532, 0, 0, 0, 0, 0, 0, 0, 0, 527, 562, 497, 532, 500, 511, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 528, 536, 485, 501, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 508, 520, 543, 439, 530, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 520, 540, 501, 487, 529, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4029, 4072, 4136, 4135, 4062, 4077, 4149] avg=4096.0000 max=4149 max_rank=7 min=4029 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[566, 488, 504, 490, 537, 502, 499, 511, 38, 0, 0, 0, 0, 0, 0, 0, 515, 483, 535, 524, 498, 561, 467, 486, 16, 5, 4, 3, 0, 0, 0, 0, 500, 481, 542, 529, 493, 462, 515, 492, 21, 11, 0, 0, 0, 0, 0, 0, 490, 517, 544, 532, 498, 482, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 558, 566, 528, 517, 505, 514, 474, 515, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 490, 549, 533, 502, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 548, 497, 504, 505, 475, 540, 504, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 539, 522, 507, 483, 486, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4097, 4046, 4057, 4177, 4137, 4087, 4032] avg=4096.0000 max=4177 max_rank=4 min=4032 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[477, 523, 487, 482, 549, 513, 508, 509, 40, 0, 0, 0, 0, 0, 0, 0, 506, 489, 513, 518, 475, 521, 552, 510, 17, 5, 4, 3, 0, 0, 0, 0, 493, 548, 520, 479, 568, 520, 505, 517, 22, 11, 0, 0, 0, 0, 0, 0, 526, 518, 479, 487, 480, 494, 502, 536, 0, 0, 0, 0, 0, 0, 0, 0, 496, 499, 496, 513, 556, 515, 502, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 520, 542, 516, 519, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 540, 553, 504, 504, 517, 485, 534, 486, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 530, 512, 474, 500, 502, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4113, 4183, 4022, 4101, 4103, 4123, 4035] avg=4096.0000 max=4183 max_rank=2 min=4022 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 481, 502, 485, 486, 501, 528, 36, 0, 0, 0, 0, 0, 0, 0, 492, 525, 490, 516, 500, 515, 543, 500, 18, 5, 4, 3, 0, 0, 0, 0, 505, 484, 468, 516, 481, 519, 504, 519, 20, 10, 0, 0, 0, 0, 0, 0, 532, 559, 505, 472, 520, 504, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 529, 540, 519, 488, 454, 566, 0, 0, 0, 0, 0, 0, 0, 0, 517, 500, 515, 529, 526, 541, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 539, 521, 468, 467, 464, 521, 0, 0, 0, 0, 0, 0, 0, 0, 527, 479, 528, 553, 525, 535, 504, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4111, 4026, 4092, 4156, 4142, 4014, 4185] avg=4096.0000 max=4185 max_rank=7 min=4014 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 529, 543, 492, 520, 484, 511, 505, 37, 0, 0, 0, 0, 0, 0, 0, 485, 544, 517, 507, 512, 512, 517, 518, 18, 5, 4, 3, 0, 0, 0, 0, 498, 497, 492, 479, 542, 502, 530, 518, 21, 11, 0, 0, 0, 0, 0, 0, 544, 530, 554, 457, 518, 544, 519, 538, 0, 0, 0, 0, 0, 0, 0, 0, 477, 535, 497, 537, 525, 517, 463, 474, 0, 0, 0, 0, 0, 0, 0, 0, 516, 495, 487, 495, 502, 510, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0, 509, 544, 494, 475, 511, 535, 473, 499, 0, 0, 0, 0, 0, 0, 0, 0, 488, 495, 513, 509, 553, 510, 536, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4142, 4090, 4204, 4025, 4021, 4040, 4105] avg=4096.0000 max=4204 max_rank=3 min=4021 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[523, 500, 505, 498, 499, 515, 518, 510, 42, 0, 0, 0, 0, 0, 0, 0, 502, 480, 521, 483, 511, 467, 528, 500, 21, 9, 8, 8, 0, 0, 0, 0, 492, 523, 513, 504, 526, 506, 530, 491, 25, 15, 0, 0, 0, 0, 0, 0, 497, 509, 540, 504, 534, 517, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 528, 502, 524, 519, 522, 505, 445, 498, 0, 0, 0, 0, 0, 0, 0, 0, 566, 530, 510, 503, 491, 509, 487, 530, 0, 0, 0, 0, 0, 0, 0, 0, 553, 518, 484, 509, 506, 531, 529, 547, 0, 0, 0, 0, 0, 0, 0, 0, 495, 475, 522, 496, 522, 494, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4038, 4125, 4135, 4043, 4126, 4177, 4014] avg=4096.0000 max=4177 max_rank=6 min=4014 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[486, 514, 460, 518, 488, 532, 527, 510, 39, 0, 0, 0, 0, 0, 0, 0, 509, 488, 538, 501, 463, 486, 544, 504, 16, 5, 4, 3, 0, 0, 0, 0, 520, 492, 561, 528, 510, 531, 519, 500, 22, 11, 0, 0, 0, 0, 0, 0, 494, 529, 535, 508, 535, 511, 523, 486, 0, 0, 0, 0, 0, 0, 0, 0, 546, 484, 529, 530, 495, 504, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 507, 493, 517, 489, 482, 518, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 502, 512, 499, 522, 484, 530, 0, 0, 0, 0, 0, 0, 0, 0, 535, 505, 492, 531, 521, 522, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4061, 4194, 4121, 4102, 3998, 4083, 4135] avg=4096.0000 max=4194 max_rank=2 min=3998 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 256, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4352, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[550, 519, 515, 536, 530, 520, 513, 528, 495, 524, 520, 494, 515, 510, 474, 492, 503, 493, 535, 526, 533, 510, 536, 489, 495, 531, 546, 531, 494, 519, 461, 504, 536, 503, 514, 497, 483, 484, 520, 463, 487, 515, 521, 505, 532, 510, 488, 506, 515, 491, 520, 515, 536, 492, 500, 503, 560, 511, 537, 524, 511, 479, 547, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[485, 517, 482, 501, 511, 493, 542, 510, 509, 522, 496, 503, 556, 512, 513, 527, 525, 492, 549, 520, 487, 463, 495, 507, 496, 485, 524, 525, 520, 511, 542, 499, 546, 519, 519, 513, 507, 506, 492, 519, 469, 504, 549, 506, 517, 526, 513, 482, 523, 526, 525, 525, 500, 496, 536, 530, 490, 522, 513, 490, 531, 512, 515, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[532, 503, 471, 515, 519, 490, 476, 499, 531, 495, 492, 526, 506, 515, 503, 524, 517, 525, 485, 540, 520, 507, 480, 523, 509, 513, 522, 534, 477, 542, 476, 538, 520, 503, 481, 513, 534, 522, 526, 575, 517, 513, 534, 515, 541, 513, 501, 511, 526, 490, 500, 526, 498, 519, 519, 522, 508, 507, 489, 480, 511, 526, 509, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[518, 479, 507, 474, 545, 537, 483, 518, 515, 528, 505, 507, 494, 498, 522, 496, 487, 538, 523, 498, 506, 502, 495, 530, 519, 530, 514, 514, 488, 472, 508, 532, 539, 506, 499, 515, 499, 536, 526, 552, 530, 511, 547, 506, 515, 482, 543, 526, 499, 531, 472, 501, 484, 527, 508, 532, 518, 489, 495, 481, 532, 522, 525, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[536, 540, 469, 476, 509, 556, 540, 464, 525, 481, 531, 517, 489, 522, 509, 534, 511, 478, 484, 500, 481, 533, 498, 502, 494, 512, 518, 499, 469, 525, 576, 536, 511, 490, 479, 502, 514, 519, 502, 482, 518, 497, 503, 514, 528, 552, 516, 524, 536, 509, 527, 515, 477, 530, 545, 496, 508, 555, 520, 532, 505, 497, 532, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[503, 494, 515, 514, 506, 509, 499, 499, 480, 504, 492, 522, 504, 528, 527, 538, 522, 499, 528, 501, 524, 491, 511, 505, 533, 527, 497, 558, 484, 516, 467, 491, 495, 549, 516, 526, 487, 528, 513, 502, 470, 505, 533, 501, 499, 487, 498, 512, 543, 528, 517, 548, 509, 504, 468, 544, 523, 506, 549, 531, 502, 532, 520, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[472, 505, 498, 490, 508, 506, 553, 507, 465, 488, 527, 508, 518, 488, 499, 508, 494, 508, 524, 524, 518, 544, 510, 476, 533, 488, 491, 512, 530, 519, 516, 505, 535, 512, 517, 539, 530, 514, 471, 522, 500, 481, 505, 519, 517, 513, 545, 548, 525, 513, 544, 539, 492, 493, 537, 498, 522, 530, 510, 551, 520, 513, 485, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[518, 505, 551, 491, 555, 549, 460, 500, 546, 490, 503, 548, 482, 503, 482, 531, 518, 493, 512, 510, 540, 508, 511, 530, 530, 491, 519, 501, 509, 482, 530, 528, 494, 518, 468, 568, 484, 537, 527, 499, 500, 519, 544, 521, 496, 502, 504, 536, 517, 510, 513, 498, 482, 527, 488, 526, 521, 496, 485, 500, 545, 503, 491, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[532, 503, 471, 515, 519, 490, 476, 499, 11, 8, 0, 0, 0, 0, 0, 0, 531, 495, 492, 526, 506, 515, 503, 524, 19, 0, 0, 0, 0, 0, 0, 0, 517, 525, 485, 540, 520, 507, 480, 523, 9, 4, 3, 0, 0, 0, 0, 0, 509, 513, 522, 534, 477, 542, 476, 538, 1, 0, 0, 0, 0, 0, 0, 0, 515, 503, 481, 513, 534, 522, 526, 575, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 525, 515, 541, 513, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 500, 526, 498, 519, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 489, 480, 511, 526, 509, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4111, 4113, 4112, 4169, 4136, 4086, 4017] avg=4096.0000 max=4169 max_rank=4 min=4017 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[485, 517, 482, 501, 511, 493, 542, 510, 11, 8, 0, 0, 0, 0, 0, 0, 509, 522, 496, 503, 556, 512, 513, 527, 20, 0, 0, 0, 0, 0, 0, 0, 525, 492, 549, 520, 487, 463, 495, 507, 9, 4, 3, 0, 0, 0, 0, 0, 496, 485, 524, 525, 520, 511, 542, 499, 1, 0, 0, 0, 0, 0, 0, 0, 541, 519, 519, 513, 507, 506, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 469, 504, 540, 506, 517, 526, 513, 482, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 525, 525, 500, 496, 536, 530, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 513, 490, 531, 512, 515, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4158, 4054, 4103, 4116, 4057, 4147, 4073] avg=4096.0000 max=4158 max_rank=1 min=4054 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[550, 519, 515, 536, 530, 520, 513, 528, 10, 8, 0, 0, 0, 0, 0, 0, 495, 524, 520, 494, 515, 510, 474, 492, 20, 0, 0, 0, 0, 0, 0, 0, 503, 493, 535, 526, 533, 510, 536, 489, 9, 4, 3, 0, 0, 0, 0, 0, 495, 531, 546, 531, 494, 519, 461, 504, 1, 0, 0, 0, 0, 0, 0, 0, 531, 503, 514, 497, 483, 484, 520, 463, 0, 0, 0, 0, 0, 0, 0, 0, 487, 515, 512, 505, 532, 510, 488, 506, 0, 0, 0, 0, 0, 0, 0, 0, 502, 491, 520, 515, 536, 492, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0, 560, 511, 537, 524, 511, 479, 547, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4229, 4044, 4141, 4082, 3995, 4055, 4059, 4163] avg=4096.0000 max=4229 max_rank=0 min=3995 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 479, 507, 474, 545, 537, 483, 518, 10, 8, 0, 0, 0, 0, 0, 0, 515, 528, 505, 507, 494, 498, 522, 496, 20, 0, 0, 0, 0, 0, 0, 0, 487, 538, 523, 498, 506, 502, 495, 530, 9, 4, 3, 0, 0, 0, 0, 0, 519, 530, 514, 514, 488, 472, 508, 532, 1, 0, 0, 0, 0, 0, 0, 0, 534, 506, 499, 515, 499, 536, 526, 552, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 538, 506, 515, 482, 543, 526, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 472, 501, 484, 527, 508, 532, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 495, 481, 532, 522, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4085, 4095, 4078, 4167, 4151, 4041, 4072] avg=4096.0000 max=4167 max_rank=4 min=4041 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[536, 540, 469, 476, 509, 556, 540, 464, 11, 8, 0, 0, 0, 0, 0, 0, 525, 481, 531, 517, 489, 522, 509, 534, 19, 0, 0, 0, 0, 0, 0, 0, 511, 478, 484, 500, 481, 533, 498, 502, 8, 4, 3, 0, 0, 0, 0, 0, 494, 512, 518, 499, 469, 525, 576, 536, 1, 0, 0, 0, 0, 0, 0, 0, 506, 490, 479, 502, 514, 519, 502, 482, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 495, 514, 528, 552, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 527, 515, 477, 530, 545, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 555, 520, 532, 505, 497, 532, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4127, 4002, 4130, 3994, 4144, 4121, 4141] avg=4096.0000 max=4144 max_rank=5 min=3994 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 494, 515, 514, 506, 509, 499, 499, 11, 8, 0, 0, 0, 0, 0, 0, 480, 504, 492, 522, 504, 528, 527, 538, 20, 0, 0, 0, 0, 0, 0, 0, 522, 499, 528, 501, 524, 491, 511, 505, 9, 4, 3, 0, 0, 0, 0, 0, 533, 527, 497, 558, 484, 516, 467, 491, 1, 0, 0, 0, 0, 0, 0, 0, 490, 549, 516, 526, 487, 528, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 470, 505, 524, 501, 499, 487, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 517, 548, 509, 504, 468, 544, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 549, 531, 502, 532, 520, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4115, 4097, 4074, 4111, 3996, 4147, 4170] avg=4096.0000 max=4170 max_rank=7 min=3996 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[472, 505, 498, 490, 508, 506, 553, 507, 14, 9, 0, 0, 0, 0, 0, 0, 465, 488, 527, 508, 518, 488, 499, 508, 22, 0, 0, 0, 0, 0, 0, 0, 494, 508, 524, 524, 518, 544, 510, 476, 12, 10, 8, 0, 0, 0, 0, 0, 533, 488, 491, 512, 530, 519, 516, 505, 4, 0, 0, 0, 0, 0, 0, 0, 521, 512, 517, 539, 530, 514, 471, 522, 0, 0, 0, 0, 0, 0, 0, 0, 500, 481, 493, 519, 517, 513, 545, 548, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 544, 539, 492, 493, 537, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 530, 510, 551, 520, 513, 485, 465, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4023, 4128, 4098, 4126, 4116, 4119, 4096] avg=4096.0000 max=4128 max_rank=2 min=4023 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 505, 551, 491, 555, 549, 460, 500, 10, 8, 0, 0, 0, 0, 0, 0, 546, 490, 503, 548, 482, 503, 482, 531, 20, 0, 0, 0, 0, 0, 0, 0, 518, 493, 512, 510, 540, 508, 511, 530, 9, 4, 3, 0, 0, 0, 0, 0, 530, 491, 519, 501, 509, 482, 530, 528, 1, 0, 0, 0, 0, 0, 0, 0, 489, 518, 468, 568, 484, 537, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 535, 521, 496, 502, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 513, 498, 482, 527, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 521, 496, 485, 500, 545, 503, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4105, 4138, 4091, 4090, 4113, 4048, 4036] avg=4096.0000 max=4147 max_rank=0 min=4036 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 15:43:12] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 277.3 | throughput per GPU (TFLOP/s/GPU): 445.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.517654E+00 | loss scale: 1.0 | grad norm: 30.447 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[573, 511, 503, 499, 508, 515, 513, 508, 524, 496, 497, 531, 453, 485, 505, 535, 494, 522, 513, 539, 496, 505, 512, 558, 509, 489, 471, 495, 479, 500, 558, 534, 521, 501, 504, 502, 536, 529, 512, 517, 503, 507, 533, 514, 512, 509, 525, 500, 530, 552, 491, 525, 565, 480, 490, 522, 522, 489, 502, 511, 475, 535, 529, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[518, 507, 504, 511, 454, 514, 511, 512, 504, 514, 508, 483, 536, 569, 499, 472, 487, 468, 545, 519, 516, 495, 508, 498, 527, 525, 541, 531, 514, 522, 504, 547, 500, 480, 508, 522, 504, 561, 510, 494, 526, 529, 535, 532, 495, 497, 544, 493, 484, 505, 522, 515, 509, 496, 511, 521, 519, 497, 488, 497, 531, 546, 534, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[494, 534, 523, 505, 494, 491, 487, 501, 506, 522, 483, 466, 511, 504, 514, 511, 521, 504, 499, 502, 506, 519, 494, 524, 512, 516, 509, 503, 507, 556, 516, 554, 519, 493, 531, 540, 484, 518, 489, 532, 509, 551, 491, 542, 548, 486, 504, 550, 527, 536, 509, 485, 506, 504, 549, 514, 482, 488, 493, 507, 539, 514, 509, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[526, 522, 534, 495, 499, 513, 517, 517, 505, 530, 516, 506, 549, 509, 522, 522, 483, 483, 544, 521, 535, 514, 485, 452, 551, 540, 511, 539, 509, 488, 493, 516, 515, 499, 531, 474, 515, 530, 515, 492, 504, 510, 507, 539, 479, 497, 525, 489, 522, 523, 525, 503, 498, 495, 520, 486, 502, 527, 514, 493, 518, 520, 544, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[521, 510, 480, 520, 528, 513, 510, 506, 481, 493, 540, 556, 504, 516, 512, 510, 516, 500, 551, 566, 502, 522, 495, 480, 547, 507, 523, 496, 530, 481, 538, 490, 511, 508, 498, 506, 528, 518, 514, 551, 517, 546, 489, 548, 529, 550, 505, 494, 467, 484, 500, 530, 490, 495, 485, 497, 506, 504, 524, 493, 535, 461, 547, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[496, 543, 543, 485, 516, 506, 528, 521, 477, 463, 486, 511, 506, 497, 531, 511, 561, 528, 533, 512, 509, 532, 486, 477, 505, 494, 445, 525, 541, 545, 540, 526, 512, 520, 518, 502, 542, 554, 565, 514, 485, 486, 488, 491, 543, 523, 486, 545, 511, 522, 519, 491, 502, 518, 513, 513, 506, 449, 517, 488, 496, 518, 514, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[475, 541, 493, 504, 534, 458, 518, 490, 489, 497, 529, 483, 545, 512, 529, 527, 495, 486, 503, 506, 526, 515, 524, 481, 525, 537, 520, 542, 527, 510, 518, 526, 531, 545, 484, 497, 511, 493, 551, 516, 531, 540, 500, 453, 564, 530, 539, 525, 494, 518, 515, 492, 502, 529, 544, 464, 477, 516, 524, 516, 496, 520, 510, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[509, 486, 497, 485, 489, 515, 474, 531, 526, 514, 540, 538, 523, 484, 565, 508, 499, 545, 510, 495, 508, 464, 519, 536, 528, 531, 526, 536, 507, 497, 490, 505, 494, 514, 502, 518, 519, 494, 508, 550, 503, 559, 509, 493, 503, 466, 518, 537, 484, 480, 507, 558, 513, 541, 520, 558, 491, 511, 504, 501, 466, 493, 529, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 504, 511, 454, 514, 511, 512, 14, 14, 0, 0, 0, 0, 0, 0, 504, 514, 508, 483, 536, 569, 499, 472, 10, 0, 0, 0, 0, 0, 0, 0, 487, 468, 545, 519, 516, 495, 508, 498, 12, 3, 0, 0, 0, 0, 0, 0, 483, 525, 541, 531, 514, 522, 504, 547, 0, 0, 0, 0, 0, 0, 0, 0, 500, 480, 508, 522, 504, 536, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 535, 532, 495, 497, 544, 493, 0, 0, 0, 0, 0, 0, 0, 0, 484, 505, 522, 515, 509, 496, 511, 521, 12, 0, 0, 0, 0, 0, 0, 0, 519, 497, 488, 497, 531, 546, 534, 500, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4095, 4051, 4167, 4054, 4125, 4075, 4142] avg=4096.0000 max=4167 max_rank=3 min=4051 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[573, 511, 503, 499, 508, 515, 513, 508, 13, 13, 0, 0, 0, 0, 0, 0, 524, 496, 497, 531, 453, 485, 505, 535, 9, 0, 0, 0, 0, 0, 0, 0, 494, 522, 513, 539, 496, 505, 512, 558, 11, 3, 0, 0, 0, 0, 0, 0, 467, 489, 471, 495, 479, 500, 558, 534, 0, 0, 0, 0, 0, 0, 0, 0, 521, 501, 504, 502, 536, 506, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 533, 514, 512, 509, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 530, 552, 491, 525, 565, 480, 490, 522, 11, 0, 0, 0, 0, 0, 0, 0, 522, 489, 502, 511, 475, 535, 529, 495, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4035, 4153, 3993, 4099, 4079, 4166, 4087] avg=4096.0000 max=4166 max_rank=6 min=3993 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[521, 510, 480, 520, 528, 513, 510, 506, 14, 14, 0, 0, 0, 0, 0, 0, 481, 493, 540, 556, 504, 516, 512, 510, 9, 0, 0, 0, 0, 0, 0, 0, 516, 500, 551, 566, 502, 522, 495, 480, 12, 3, 0, 0, 0, 0, 0, 0, 502, 507, 523, 496, 530, 481, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 498, 506, 528, 495, 514, 551, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 489, 548, 529, 550, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 467, 484, 500, 530, 490, 495, 485, 497, 11, 0, 0, 0, 0, 0, 0, 0, 506, 504, 524, 493, 535, 461, 547, 494, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4121, 4147, 4067, 4111, 4152, 3959, 4095] avg=4096.0000 max=4152 max_rank=5 min=3959 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 522, 534, 495, 499, 513, 517, 517, 13, 15, 0, 0, 0, 0, 0, 0, 505, 530, 516, 506, 549, 509, 522, 522, 9, 0, 0, 0, 0, 0, 0, 0, 483, 483, 544, 521, 535, 514, 485, 452, 11, 3, 0, 0, 0, 0, 0, 0, 505, 540, 511, 539, 509, 488, 493, 516, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 531, 474, 515, 507, 515, 492, 0, 0, 0, 0, 0, 0, 0, 0, 504, 486, 507, 539, 479, 497, 525, 489, 0, 0, 0, 0, 0, 0, 0, 0, 522, 523, 525, 503, 498, 495, 520, 486, 11, 0, 0, 0, 0, 0, 0, 0, 502, 527, 514, 493, 518, 520, 544, 511, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4168, 4031, 4101, 4048, 4026, 4083, 4160] avg=4096.0000 max=4168 max_rank=1 min=4026 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 534, 523, 505, 494, 491, 487, 501, 14, 14, 0, 0, 0, 0, 0, 0, 506, 522, 483, 466, 511, 504, 514, 511, 9, 0, 0, 0, 0, 0, 0, 0, 521, 504, 499, 502, 506, 519, 494, 524, 12, 3, 0, 0, 0, 0, 0, 0, 469, 516, 509, 503, 507, 556, 516, 554, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 531, 540, 484, 495, 489, 532, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 491, 542, 548, 486, 504, 550, 0, 0, 0, 0, 0, 0, 0, 0, 527, 536, 509, 485, 506, 504, 549, 514, 11, 0, 0, 0, 0, 0, 0, 0, 482, 488, 493, 507, 539, 514, 509, 531, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4026, 4084, 4130, 4083, 4155, 4141, 4092] avg=4096.0000 max=4155 max_rank=5 min=4026 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[475, 541, 493, 504, 534, 458, 518, 490, 14, 14, 0, 0, 0, 0, 0, 0, 489, 497, 529, 483, 545, 512, 529, 527, 9, 0, 0, 0, 0, 0, 0, 0, 495, 486, 503, 506, 526, 515, 524, 481, 12, 3, 0, 0, 0, 0, 0, 0, 481, 537, 520, 542, 527, 510, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 531, 545, 484, 497, 511, 471, 551, 516, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 500, 453, 564, 530, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 515, 492, 502, 529, 544, 464, 10, 0, 0, 0, 0, 0, 0, 0, 477, 516, 524, 516, 496, 520, 510, 476, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4120, 4051, 4161, 4106, 4156, 4068, 4065] avg=4096.0000 max=4161 max_rank=3 min=4041 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[496, 543, 543, 485, 516, 506, 528, 521, 18, 17, 0, 0, 0, 0, 0, 0, 477, 463, 486, 511, 506, 497, 531, 511, 14, 0, 0, 0, 0, 0, 0, 0, 561, 528, 533, 512, 509, 532, 486, 477, 15, 7, 0, 0, 0, 0, 0, 0, 457, 494, 445, 525, 541, 545, 540, 526, 0, 0, 0, 0, 0, 0, 0, 0, 512, 520, 518, 502, 542, 517, 565, 514, 0, 0, 0, 0, 0, 0, 0, 0, 485, 453, 488, 491, 543, 523, 486, 545, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 519, 491, 502, 518, 513, 513, 16, 0, 0, 0, 0, 0, 0, 0, 506, 449, 517, 488, 496, 518, 514, 538, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 3996, 4160, 4073, 4190, 4014, 4105, 4057] avg=4096.0000 max=4190 max_rank=4 min=3996 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 486, 497, 485, 489, 515, 474, 531, 15, 14, 0, 0, 0, 0, 0, 0, 526, 514, 540, 538, 523, 484, 565, 508, 9, 0, 0, 0, 0, 0, 0, 0, 499, 545, 510, 495, 508, 464, 519, 536, 12, 3, 0, 0, 0, 0, 0, 0, 484, 531, 526, 536, 507, 497, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 502, 518, 519, 472, 508, 550, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 509, 493, 503, 466, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 484, 480, 507, 558, 513, 541, 520, 558, 10, 0, 0, 0, 0, 0, 0, 0, 491, 511, 504, 501, 466, 493, 529, 545, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4207, 4091, 4076, 4077, 4061, 4171, 4070] avg=4096.0000 max=4207 max_rank=1 min=4015 min_rank=0 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[538, 503, 484, 494, 496, 491, 479, 514, 507, 519, 488, 526, 475, 487, 496, 535, 516, 528, 526, 522, 527, 513, 502, 509, 532, 513, 511, 533, 521, 533, 523, 534, 508, 507, 520, 496, 526, 511, 533, 510, 507, 551, 502, 469, 533, 482, 522, 529, 496, 507, 513, 521, 524, 521, 497, 492, 529, 507, 530, 529, 509, 487, 494, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[546, 490, 516, 489, 503, 524, 501, 506, 499, 557, 531, 536, 484, 473, 527, 526, 527, 490, 505, 504, 513, 539, 502, 526, 466, 498, 549, 492, 508, 494, 492, 500, 510, 489, 514, 547, 536, 488, 522, 506, 536, 478, 477, 491, 500, 508, 543, 523, 479, 533, 492, 536, 518, 487, 529, 532, 511, 489, 534, 500, 541, 537, 526, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[546, 501, 553, 497, 490, 536, 527, 501, 509, 533, 510, 487, 490, 498, 526, 507, 482, 546, 546, 496, 543, 483, 546, 515, 515, 485, 486, 543, 512, 507, 495, 530, 459, 481, 525, 523, 506, 525, 515, 523, 547, 488, 538, 530, 517, 506, 510, 508, 547, 520, 505, 491, 503, 537, 510, 531, 487, 474, 481, 517, 533, 477, 497, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[523, 545, 518, 505, 517, 490, 555, 493, 506, 495, 498, 509, 505, 472, 522, 517, 517, 518, 510, 497, 504, 527, 504, 532, 525, 523, 534, 518, 481, 542, 507, 495, 543, 522, 510, 551, 499, 477, 505, 474, 523, 533, 526, 539, 470, 529, 509, 510, 522, 484, 518, 500, 531, 487, 533, 539, 493, 485, 534, 535, 497, 459, 540, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[506, 526, 491, 514, 517, 469, 521, 510, 512, 557, 519, 526, 503, 507, 477, 544, 527, 511, 500, 520, 530, 543, 551, 489, 480, 484, 509, 510, 490, 522, 479, 478, 495, 506, 496, 537, 510, 502, 520, 524, 513, 548, 470, 555, 530, 531, 522, 521, 534, 552, 451, 514, 512, 505, 527, 486, 470, 510, 511, 537, 503, 484, 528, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[511, 498, 521, 529, 505, 538, 478, 558, 479, 508, 535, 503, 496, 531, 512, 486, 483, 510, 494, 545, 520, 523, 493, 499, 536, 541, 476, 509, 509, 509, 546, 521, 504, 533, 531, 513, 492, 540, 515, 498, 530, 539, 555, 485, 489, 525, 503, 480, 520, 512, 530, 508, 506, 501, 510, 487, 515, 547, 510, 526, 481, 479, 477, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[524, 532, 507, 546, 511, 526, 502, 504, 472, 487, 524, 520, 500, 518, 510, 513, 528, 500, 482, 513, 510, 542, 530, 511, 518, 540, 491, 536, 483, 467, 522, 514, 491, 538, 491, 526, 478, 516, 499, 482, 515, 521, 507, 495, 523, 491, 527, 500, 521, 499, 541, 505, 500, 519, 555, 540, 500, 504, 496, 517, 572, 527, 500, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[513, 525, 536, 528, 511, 535, 534, 498, 522, 516, 469, 532, 509, 529, 500, 500, 508, 487, 531, 506, 527, 499, 499, 539, 488, 528, 526, 535, 527, 519, 498, 497, 528, 488, 538, 513, 483, 485, 538, 513, 510, 513, 507, 529, 470, 443, 505, 538, 511, 510, 503, 520, 514, 494, 520, 506, 515, 520, 477, 542, 506, 515, 526, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[524, 503, 484, 494, 496, 491, 479, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 488, 526, 475, 487, 496, 535, 24, 0, 0, 0, 0, 0, 0, 0, 516, 528, 526, 522, 503, 513, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 532, 513, 511, 533, 521, 533, 523, 534, 6, 3, 0, 0, 0, 0, 0, 0, 508, 507, 520, 496, 526, 511, 533, 510, 6, 3, 0, 0, 0, 0, 0, 0, 501, 551, 502, 469, 533, 482, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 496, 507, 513, 521, 524, 521, 488, 492, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 530, 529, 509, 487, 494, 531, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4057, 4119, 4209, 4120, 4089, 4062, 4127] avg=4096.0000 max=4209 max_rank=3 min=3985 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 501, 553, 497, 490, 536, 527, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 510, 487, 490, 498, 526, 507, 25, 0, 0, 0, 0, 0, 0, 0, 482, 546, 546, 496, 518, 483, 546, 515, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 486, 543, 512, 507, 495, 530, 7, 3, 0, 0, 0, 0, 0, 0, 459, 481, 525, 523, 506, 525, 515, 523, 6, 4, 0, 0, 0, 0, 0, 0, 540, 488, 538, 530, 517, 506, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 505, 491, 503, 537, 501, 531, 0, 0, 0, 0, 0, 0, 0, 0, 487, 474, 481, 517, 533, 477, 497, 516, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4085, 4132, 4083, 4067, 4137, 4135, 3993] avg=4096.0000 max=4137 max_rank=5 min=3993 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 526, 491, 514, 517, 469, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 512, 557, 519, 526, 503, 507, 477, 544, 25, 0, 0, 0, 0, 0, 0, 0, 527, 511, 500, 520, 505, 543, 551, 489, 0, 0, 0, 0, 0, 0, 0, 0, 480, 484, 509, 510, 490, 522, 479, 478, 6, 3, 0, 0, 0, 0, 0, 0, 495, 506, 496, 537, 510, 502, 520, 524, 6, 3, 0, 0, 0, 0, 0, 0, 507, 548, 470, 555, 530, 531, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 534, 552, 451, 514, 512, 505, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 470, 510, 511, 537, 503, 484, 528, 542, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4170, 4146, 3961, 4099, 4184, 4072, 4095] avg=4096.0000 max=4184 max_rank=5 min=3961 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 498, 521, 529, 505, 538, 478, 558, 0, 0, 0, 0, 0, 0, 0, 0, 479, 508, 535, 503, 496, 531, 512, 486, 24, 0, 0, 0, 0, 0, 0, 0, 483, 510, 494, 545, 496, 523, 493, 499, 0, 0, 0, 0, 0, 0, 0, 0, 536, 541, 476, 509, 509, 509, 546, 521, 7, 3, 0, 0, 0, 0, 0, 0, 504, 533, 531, 513, 492, 540, 515, 498, 6, 3, 0, 0, 0, 0, 0, 0, 523, 539, 555, 485, 489, 525, 503, 480, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 530, 508, 506, 501, 501, 487, 0, 0, 0, 0, 0, 0, 0, 0, 515, 547, 510, 526, 481, 479, 477, 525, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4074, 4043, 4157, 4135, 4099, 4065, 4071] avg=4096.0000 max=4157 max_rank=3 min=4043 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 490, 516, 489, 503, 524, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 499, 557, 531, 536, 484, 473, 527, 526, 24, 0, 0, 0, 0, 0, 0, 0, 527, 490, 505, 504, 489, 539, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 466, 498, 549, 492, 508, 494, 492, 500, 7, 3, 0, 0, 0, 0, 0, 0, 510, 489, 514, 547, 536, 488, 522, 506, 6, 4, 0, 0, 0, 0, 0, 0, 529, 478, 477, 491, 500, 508, 543, 523, 0, 0, 0, 0, 0, 0, 0, 0, 479, 533, 492, 536, 518, 487, 520, 532, 0, 0, 0, 0, 0, 0, 0, 0, 511, 489, 534, 500, 541, 537, 526, 543, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4157, 4082, 4009, 4122, 4049, 4097, 4192] avg=4096.0000 max=4192 max_rank=7 min=4009 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 545, 518, 505, 517, 490, 555, 493, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 498, 509, 505, 472, 522, 517, 27, 5, 0, 0, 0, 0, 0, 0, 517, 518, 510, 497, 477, 527, 504, 532, 0, 0, 0, 0, 0, 0, 0, 0, 525, 523, 534, 518, 481, 542, 507, 495, 11, 6, 0, 0, 0, 0, 0, 0, 543, 522, 510, 551, 499, 477, 505, 474, 10, 8, 0, 0, 0, 0, 0, 0, 512, 533, 526, 539, 470, 529, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 518, 500, 531, 487, 517, 539, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 534, 535, 497, 459, 540, 487, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4056, 4082, 4142, 4099, 4128, 4098, 4045] avg=4096.0000 max=4142 max_rank=3 min=4045 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 532, 507, 546, 511, 526, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 472, 487, 524, 520, 500, 518, 510, 513, 24, 0, 0, 0, 0, 0, 0, 0, 528, 500, 482, 513, 486, 542, 530, 511, 0, 0, 0, 0, 0, 0, 0, 0, 518, 540, 491, 536, 483, 467, 522, 514, 6, 3, 0, 0, 0, 0, 0, 0, 491, 538, 491, 526, 478, 516, 499, 482, 7, 3, 0, 0, 0, 0, 0, 0, 509, 521, 507, 495, 523, 491, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 541, 505, 500, 519, 545, 540, 0, 0, 0, 0, 0, 0, 0, 0, 500, 504, 496, 517, 572, 527, 500, 489, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4068, 4092, 4080, 4031, 4073, 4170, 4116] avg=4096.0000 max=4170 max_rank=6 min=4031 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 525, 536, 528, 511, 535, 534, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 469, 532, 509, 529, 500, 500, 24, 0, 0, 0, 0, 0, 0, 0, 508, 487, 531, 506, 503, 499, 499, 539, 0, 0, 0, 0, 0, 0, 0, 0, 488, 528, 526, 535, 527, 519, 498, 497, 6, 3, 0, 0, 0, 0, 0, 0, 528, 488, 538, 513, 483, 485, 538, 513, 6, 3, 0, 0, 0, 0, 0, 0, 504, 513, 507, 529, 470, 443, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 503, 520, 514, 494, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0, 515, 520, 477, 542, 506, 515, 526, 517, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4101, 4072, 4127, 4095, 4009, 4069, 4129] avg=4096.0000 max=4166 max_rank=0 min=4009 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[505, 492, 488, 494, 482, 509, 531, 486, 517, 507, 537, 501, 492, 480, 514, 538, 526, 566, 530, 527, 491, 548, 470, 488, 499, 517, 481, 518, 563, 531, 487, 507, 552, 552, 477, 490, 524, 511, 486, 484, 503, 491, 511, 515, 531, 502, 523, 537, 516, 509, 545, 522, 499, 527, 522, 537, 503, 525, 517, 529, 492, 519, 474, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[534, 513, 505, 523, 489, 506, 507, 516, 504, 504, 526, 500, 523, 516, 506, 531, 492, 504, 480, 541, 505, 506, 541, 494, 509, 495, 515, 527, 523, 507, 492, 542, 508, 502, 469, 492, 478, 512, 509, 488, 506, 542, 533, 501, 493, 520, 530, 535, 509, 501, 520, 506, 543, 499, 516, 480, 531, 505, 541, 550, 513, 522, 532, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[506, 510, 520, 493, 496, 531, 502, 532, 493, 490, 541, 536, 525, 486, 476, 514, 526, 507, 509, 525, 497, 543, 501, 529, 515, 546, 498, 515, 516, 513, 509, 532, 518, 503, 493, 536, 515, 498, 507, 514, 535, 527, 526, 534, 514, 538, 506, 530, 504, 486, 523, 497, 482, 497, 501, 486, 538, 502, 530, 536, 482, 508, 481, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[518, 541, 520, 558, 512, 532, 536, 531, 441, 525, 529, 543, 528, 495, 539, 543, 515, 539, 543, 487, 519, 504, 517, 507, 522, 528, 503, 504, 511, 494, 512, 509, 500, 512, 537, 494, 517, 536, 531, 517, 495, 503, 473, 523, 508, 521, 502, 431, 515, 488, 485, 506, 495, 522, 538, 525, 517, 515, 481, 519, 541, 488, 477, 451] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[522, 528, 529, 533, 525, 515, 490, 528, 544, 551, 529, 525, 536, 516, 499, 519, 502, 512, 510, 550, 519, 513, 532, 553, 532, 500, 522, 477, 478, 493, 507, 542, 531, 496, 511, 505, 513, 486, 488, 536, 490, 488, 515, 515, 545, 478, 495, 532, 536, 525, 496, 488, 479, 507, 505, 530, 476, 479, 475, 493, 468, 505, 527, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[538, 520, 480, 521, 536, 519, 473, 501, 512, 504, 505, 520, 576, 521, 501, 502, 480, 523, 533, 492, 517, 499, 540, 499, 522, 531, 474, 538, 487, 494, 499, 487, 536, 501, 544, 467, 527, 523, 555, 502, 484, 483, 515, 525, 559, 514, 492, 494, 512, 468, 510, 512, 511, 468, 515, 518, 558, 504, 524, 522, 529, 509, 548, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[533, 527, 524, 532, 487, 532, 477, 534, 543, 489, 504, 510, 483, 493, 495, 502, 516, 478, 524, 481, 521, 533, 531, 495, 504, 516, 517, 493, 495, 523, 498, 557, 499, 499, 506, 524, 495, 502, 528, 509, 492, 549, 496, 502, 545, 508, 462, 544, 495, 525, 485, 529, 520, 517, 527, 512, 506, 541, 518, 515, 510, 541, 511, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[494, 504, 535, 521, 540, 482, 486, 489, 521, 511, 527, 483, 496, 463, 520, 523, 526, 498, 534, 479, 515, 510, 488, 489, 545, 506, 501, 490, 515, 556, 507, 496, 527, 537, 500, 495, 500, 537, 478, 571, 498, 498, 532, 542, 485, 481, 500, 497, 532, 550, 506, 509, 531, 501, 532, 523, 543, 530, 522, 514, 478, 533, 526, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 492, 488, 482, 482, 509, 531, 486, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 531, 501, 492, 480, 514, 538, 0, 0, 0, 0, 0, 0, 0, 0, 526, 566, 518, 527, 491, 548, 470, 488, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 481, 518, 563, 531, 487, 507, 2, 1, 0, 0, 0, 0, 0, 0, 552, 552, 477, 490, 524, 511, 486, 484, 8, 1, 0, 0, 0, 0, 0, 0, 503, 491, 511, 515, 531, 502, 523, 537, 4, 4, 0, 0, 0, 0, 0, 0, 516, 509, 545, 522, 499, 527, 522, 537, 11, 0, 0, 0, 0, 0, 0, 0, 503, 525, 517, 528, 492, 519, 474, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3975, 4080, 4134, 4106, 4085, 4121, 4188, 4079] avg=4096.0000 max=4188 max_rank=6 min=3975 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 510, 520, 482, 496, 531, 502, 532, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 534, 536, 525, 486, 476, 514, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 497, 525, 497, 543, 501, 529, 0, 0, 0, 0, 0, 0, 0, 0, 515, 546, 498, 515, 516, 513, 509, 532, 2, 1, 0, 0, 0, 0, 0, 0, 518, 503, 493, 536, 515, 498, 507, 514, 8, 1, 0, 0, 0, 0, 0, 0, 535, 527, 526, 534, 514, 538, 506, 530, 5, 4, 0, 0, 0, 0, 0, 0, 504, 486, 523, 497, 482, 497, 501, 486, 10, 0, 0, 0, 0, 0, 0, 0, 538, 502, 530, 535, 482, 508, 481, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4054, 4125, 4147, 4093, 4219, 3986, 4065] avg=4096.0000 max=4219 max_rank=5 min=3986 min_rank=6 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 513, 505, 502, 489, 506, 507, 516, 0, 0, 0, 0, 0, 0, 0, 0, 504, 504, 512, 500, 523, 516, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 463, 541, 505, 506, 541, 494, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 515, 527, 523, 507, 492, 542, 4, 3, 0, 0, 0, 0, 0, 0, 508, 502, 469, 492, 478, 512, 509, 488, 10, 5, 0, 0, 0, 0, 0, 0, 506, 542, 533, 501, 493, 520, 530, 535, 10, 7, 0, 0, 0, 0, 0, 0, 509, 501, 520, 506, 543, 499, 516, 480, 16, 0, 0, 0, 0, 0, 0, 0, 531, 505, 541, 547, 513, 522, 532, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4096, 4046, 4117, 3973, 4177, 4090, 4197] avg=4096.0000 max=4197 max_rank=7 min=3973 min_rank=4 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 504, 535, 509, 540, 482, 486, 489, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 521, 483, 496, 463, 520, 523, 0, 0, 0, 0, 0, 0, 0, 0, 526, 498, 522, 479, 515, 510, 488, 489, 0, 0, 0, 0, 0, 0, 0, 0, 545, 506, 501, 490, 515, 556, 507, 496, 2, 1, 0, 0, 0, 0, 0, 0, 527, 537, 500, 495, 500, 537, 478, 571, 8, 1, 0, 0, 0, 0, 0, 0, 498, 498, 532, 542, 485, 481, 500, 497, 4, 4, 0, 0, 0, 0, 0, 0, 532, 550, 506, 509, 531, 501, 532, 523, 11, 0, 0, 0, 0, 0, 0, 0, 543, 530, 522, 513, 478, 533, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4038, 4027, 4119, 4154, 4041, 4195, 4155] avg=4096.0000 max=4195 max_rank=6 min=4027 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[522, 528, 529, 521, 525, 515, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 544, 551, 523, 525, 536, 516, 499, 519, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 498, 550, 519, 513, 532, 553, 0, 0, 0, 0, 0, 0, 0, 0, 532, 500, 522, 477, 478, 493, 507, 542, 2, 1, 0, 0, 0, 0, 0, 0, 531, 496, 511, 505, 513, 486, 488, 536, 8, 1, 0, 0, 0, 0, 0, 0, 490, 488, 515, 515, 545, 478, 495, 532, 4, 4, 0, 0, 0, 0, 0, 0, 536, 525, 496, 488, 479, 507, 505, 530, 11, 0, 0, 0, 0, 0, 0, 0, 476, 479, 475, 492, 468, 505, 527, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4213, 4179, 4054, 4075, 4066, 4077, 3946] avg=4096.0000 max=4213 max_rank=1 min=3946 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16164 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 541, 520, 545, 512, 532, 536, 531, 0, 0, 0, 0, 0, 0, 0, 0, 441, 525, 523, 543, 528, 495, 539, 543, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 531, 487, 519, 504, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 503, 504, 511, 494, 512, 509, 2, 1, 0, 0, 0, 0, 0, 0, 500, 512, 537, 494, 517, 536, 531, 517, 8, 1, 0, 0, 0, 0, 0, 0, 495, 503, 473, 523, 508, 521, 502, 431, 4, 4, 0, 0, 0, 0, 0, 0, 515, 488, 485, 506, 495, 522, 538, 525, 12, 0, 0, 0, 0, 0, 0, 0, 517, 515, 481, 518, 541, 488, 477, 451, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 4137, 4119, 4086, 4153, 3964, 4086, 3988] avg=4096.0000 max=4235 max_rank=0 min=3964 min_rank=5 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[538, 520, 480, 509, 536, 519, 473, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 500, 520, 576, 521, 501, 502, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 521, 492, 517, 499, 540, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 474, 538, 487, 494, 499, 487, 1, 1, 0, 0, 0, 0, 0, 0, 536, 501, 544, 467, 527, 523, 555, 502, 8, 1, 0, 0, 0, 0, 0, 0, 484, 483, 515, 525, 559, 514, 492, 494, 4, 4, 0, 0, 0, 0, 0, 0, 512, 468, 510, 512, 511, 468, 515, 518, 11, 0, 0, 0, 0, 0, 0, 0, 558, 504, 524, 521, 529, 509, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4136, 4071, 4034, 4164, 4074, 4025, 4188] avg=4096.0000 max=4188 max_rank=7 min=4025 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[533, 527, 524, 520, 487, 532, 477, 534, 0, 0, 0, 0, 0, 0, 0, 0, 543, 489, 499, 510, 483, 493, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 516, 478, 512, 481, 521, 533, 531, 495, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 517, 493, 495, 523, 498, 557, 1, 1, 0, 0, 0, 0, 0, 0, 499, 499, 506, 524, 495, 502, 528, 509, 8, 1, 0, 0, 0, 0, 0, 0, 492, 549, 496, 502, 545, 508, 462, 544, 4, 4, 0, 0, 0, 0, 0, 0, 495, 525, 485, 529, 520, 517, 527, 512, 11, 0, 0, 0, 0, 0, 0, 0, 506, 541, 518, 514, 510, 541, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4014, 4067, 4105, 4071, 4106, 4121, 4150] avg=4096.0000 max=4150 max_rank=7 min=4014 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[520, 522, 478, 496, 499, 480, 517, 488, 539, 485, 495, 512, 544, 508, 498, 541, 526, 505, 486, 502, 497, 501, 499, 551, 499, 522, 543, 518, 520, 494, 517, 471, 517, 500, 545, 534, 513, 551, 521, 520, 530, 476, 531, 474, 486, 508, 511, 518, 505, 500, 531, 494, 513, 495, 539, 508, 528, 506, 488, 548, 527, 537, 523, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[534, 529, 526, 535, 538, 522, 519, 498, 499, 492, 525, 505, 499, 527, 511, 521, 504, 544, 528, 532, 496, 487, 533, 500, 483, 499, 522, 485, 469, 531, 526, 554, 532, 505, 552, 513, 545, 521, 482, 533, 516, 507, 508, 497, 473, 513, 504, 517, 522, 490, 483, 522, 486, 464, 506, 563, 519, 521, 504, 478, 498, 475, 541, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[554, 501, 495, 517, 516, 511, 547, 489, 509, 480, 536, 554, 541, 484, 509, 507, 501, 507, 498, 542, 509, 523, 487, 489, 509, 536, 501, 525, 485, 487, 483, 490, 504, 524, 516, 553, 462, 491, 492, 488, 514, 512, 509, 513, 472, 493, 521, 557, 522, 542, 515, 512, 509, 529, 548, 497, 551, 511, 496, 493, 529, 513, 543, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[519, 531, 538, 512, 524, 535, 503, 535, 486, 530, 506, 485, 549, 566, 538, 511, 493, 476, 502, 484, 471, 533, 468, 534, 497, 517, 496, 493, 549, 522, 509, 506, 504, 477, 497, 499, 524, 506, 523, 530, 544, 483, 508, 538, 522, 498, 484, 572, 476, 531, 532, 505, 487, 460, 521, 493, 526, 500, 477, 533, 480, 553, 525, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[509, 530, 482, 538, 499, 474, 533, 518, 496, 513, 523, 505, 510, 503, 509, 501, 494, 491, 520, 501, 529, 544, 526, 522, 480, 524, 486, 502, 459, 517, 481, 484, 527, 511, 519, 523, 504, 546, 501, 510, 556, 483, 532, 508, 556, 475, 533, 466, 519, 543, 527, 532, 470, 538, 531, 485, 514, 530, 521, 536, 513, 512, 506, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[504, 551, 559, 513, 515, 504, 495, 488, 547, 524, 514, 510, 471, 494, 517, 551, 538, 472, 468, 526, 556, 476, 506, 489, 530, 479, 511, 550, 489, 470, 500, 527, 475, 491, 519, 504, 532, 506, 503, 537, 508, 534, 524, 520, 522, 532, 460, 503, 534, 522, 512, 517, 534, 490, 532, 492, 532, 494, 512, 492, 512, 518, 550, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[486, 498, 502, 517, 511, 518, 500, 512, 520, 494, 501, 502, 510, 509, 487, 486, 543, 471, 460, 516, 514, 523, 498, 509, 511, 499, 516, 517, 502, 543, 524, 498, 526, 519, 504, 543, 503, 516, 526, 521, 527, 516, 551, 567, 529, 495, 467, 549, 515, 503, 529, 496, 534, 541, 488, 535, 522, 499, 526, 476, 515, 522, 515, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[496, 525, 536, 498, 495, 523, 513, 530, 502, 519, 517, 510, 537, 510, 499, 514, 457, 482, 550, 505, 560, 523, 507, 517, 501, 522, 559, 486, 505, 512, 548, 486, 503, 518, 521, 498, 519, 516, 519, 548, 517, 461, 544, 541, 532, 493, 528, 500, 498, 503, 536, 512, 500, 512, 490, 508, 492, 505, 503, 493, 497, 508, 519, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 507, 478, 496, 499, 480, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 539, 485, 495, 512, 541, 508, 498, 541, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 486, 502, 497, 501, 499, 551, 12, 8, 6, 3, 1, 0, 0, 0, 499, 522, 543, 518, 520, 494, 517, 471, 26, 9, 0, 0, 0, 0, 0, 0, 517, 500, 545, 534, 513, 551, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 531, 474, 486, 508, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 505, 500, 530, 494, 513, 495, 539, 508, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 488, 548, 527, 537, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4119, 4097, 4119, 4175, 4026, 4084, 4163] avg=4096.0000 max=4175 max_rank=4 min=3985 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[534, 514, 526, 535, 538, 522, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 492, 525, 505, 496, 527, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 504, 544, 528, 532, 496, 487, 533, 500, 13, 8, 6, 3, 1, 0, 0, 0, 483, 499, 522, 485, 469, 531, 526, 554, 27, 9, 0, 0, 0, 0, 0, 0, 532, 505, 552, 513, 545, 521, 482, 506, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 508, 497, 473, 513, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 482, 522, 486, 464, 506, 563, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 504, 478, 498, 475, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4076, 4155, 4105, 4156, 4027, 4035, 4028] avg=4096.0000 max=4186 max_rank=0 min=4027 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[554, 486, 495, 517, 516, 511, 547, 489, 0, 0, 0, 0, 0, 0, 0, 0, 509, 480, 536, 554, 538, 484, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 498, 542, 509, 523, 487, 489, 13, 8, 6, 3, 1, 0, 0, 0, 509, 536, 501, 525, 485, 487, 483, 490, 24, 9, 0, 0, 0, 0, 0, 0, 504, 524, 516, 553, 462, 491, 492, 464, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 509, 513, 472, 493, 521, 557, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 514, 512, 509, 529, 548, 497, 0, 0, 0, 0, 0, 0, 0, 0, 551, 511, 496, 493, 529, 513, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4117, 4087, 4049, 4006, 4083, 4173, 4138] avg=4096.0000 max=4173 max_rank=6 min=4006 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 534, 559, 513, 515, 504, 495, 488, 0, 0, 0, 0, 0, 0, 0, 0, 547, 524, 514, 510, 468, 494, 517, 551, 0, 0, 0, 0, 0, 0, 0, 0, 538, 472, 468, 526, 556, 476, 506, 489, 13, 8, 7, 3, 1, 0, 0, 0, 530, 479, 511, 550, 489, 470, 500, 527, 27, 10, 0, 0, 0, 0, 0, 0, 475, 491, 519, 504, 532, 506, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 500, 534, 524, 520, 522, 532, 460, 503, 0, 0, 0, 0, 0, 0, 0, 0, 534, 522, 511, 517, 534, 490, 532, 492, 0, 0, 0, 0, 0, 0, 0, 0, 532, 494, 512, 492, 512, 518, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4125, 4063, 4093, 4040, 4095, 4132, 4108] avg=4096.0000 max=4132 max_rank=6 min=4040 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 515, 482, 538, 499, 474, 533, 518, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 523, 505, 507, 503, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 520, 501, 529, 544, 526, 522, 12, 9, 6, 3, 1, 0, 0, 0, 480, 524, 486, 502, 459, 517, 481, 484, 26, 9, 0, 0, 0, 0, 0, 0, 527, 511, 519, 523, 504, 546, 501, 484, 0, 0, 0, 0, 0, 0, 0, 0, 547, 483, 532, 508, 556, 475, 533, 466, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 526, 532, 470, 538, 531, 485, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 521, 536, 513, 512, 494, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4057, 4158, 3968, 4115, 4100, 4144, 4158] avg=4096.0000 max=4158 max_rank=2 min=3968 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 538, 512, 524, 535, 503, 535, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 506, 485, 546, 566, 538, 511, 0, 0, 0, 0, 0, 0, 0, 0, 493, 476, 502, 484, 471, 533, 468, 534, 12, 9, 6, 3, 1, 0, 0, 0, 497, 517, 496, 493, 549, 522, 509, 506, 27, 9, 0, 0, 0, 0, 0, 0, 504, 477, 497, 499, 524, 506, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 535, 483, 508, 538, 522, 498, 484, 572, 0, 0, 0, 0, 0, 0, 0, 0, 476, 531, 531, 505, 487, 460, 521, 493, 0, 0, 0, 0, 0, 0, 0, 0, 526, 500, 477, 533, 480, 553, 513, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4168, 3992, 4125, 4033, 4140, 4004, 4124] avg=4096.0000 max=4182 max_rank=0 min=3992 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[486, 483, 502, 517, 511, 518, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 501, 502, 507, 509, 487, 486, 0, 0, 0, 0, 0, 0, 0, 0, 543, 471, 460, 516, 514, 523, 498, 509, 12, 8, 6, 3, 1, 0, 0, 0, 511, 499, 516, 517, 502, 543, 524, 498, 26, 9, 0, 0, 0, 0, 0, 0, 526, 519, 504, 543, 503, 516, 526, 495, 0, 0, 0, 0, 0, 0, 0, 0, 519, 516, 551, 567, 529, 495, 467, 549, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 528, 496, 534, 541, 488, 535, 0, 0, 0, 0, 0, 0, 0, 0, 522, 499, 526, 476, 515, 522, 503, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4006, 4064, 4145, 4132, 4193, 4140, 4059] avg=4096.0000 max=4193 max_rank=5 min=4006 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[496, 500, 536, 498, 495, 523, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 502, 519, 517, 510, 529, 510, 499, 514, 0, 0, 0, 0, 0, 0, 0, 0, 457, 482, 550, 505, 560, 523, 507, 517, 17, 12, 11, 8, 3, 0, 0, 0, 501, 522, 559, 486, 505, 512, 548, 486, 31, 14, 0, 0, 0, 0, 0, 0, 503, 518, 521, 498, 519, 516, 519, 517, 0, 0, 0, 0, 0, 0, 0, 0, 505, 461, 544, 541, 532, 493, 528, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 533, 512, 500, 512, 490, 508, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 503, 493, 497, 508, 502, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4100, 4152, 4164, 4111, 4104, 4056, 3990] avg=4096.0000 max=4164 max_rank=3 min=3990 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[512, 509, 539, 542, 519, 512, 543, 519, 510, 481, 527, 529, 479, 510, 541, 505, 477, 508, 510, 509, 534, 488, 513, 479, 497, 501, 516, 489, 537, 500, 493, 491, 524, 514, 521, 504, 531, 534, 472, 503, 511, 507, 506, 504, 507, 533, 550, 550, 516, 511, 503, 503, 493, 544, 514, 475, 498, 502, 544, 498, 532, 523, 531, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[526, 509, 516, 567, 521, 521, 493, 507, 506, 535, 465, 515, 532, 517, 520, 525, 474, 532, 520, 457, 501, 533, 514, 448, 500, 515, 532, 474, 529, 536, 497, 504, 533, 463, 552, 523, 505, 494, 492, 508, 491, 541, 503, 507, 501, 522, 495, 496, 498, 538, 558, 519, 520, 494, 521, 490, 495, 487, 545, 522, 524, 521, 514, 555] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[492, 477, 518, 494, 505, 535, 497, 499, 495, 553, 459, 469, 511, 536, 532, 490, 530, 544, 498, 522, 520, 524, 501, 488, 521, 531, 521, 524, 545, 504, 508, 559, 516, 518, 510, 516, 512, 518, 510, 524, 509, 519, 476, 529, 497, 497, 570, 533, 495, 515, 461, 515, 514, 500, 477, 512, 505, 567, 540, 518, 505, 480, 471, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[545, 529, 549, 552, 496, 527, 505, 541, 494, 557, 510, 514, 514, 526, 486, 511, 504, 535, 527, 539, 496, 484, 506, 508, 541, 519, 508, 475, 494, 491, 507, 510, 555, 543, 514, 527, 478, 502, 468, 509, 508, 482, 524, 481, 480, 489, 523, 473, 523, 543, 524, 516, 522, 510, 506, 522, 482, 486, 508, 502, 494, 537, 534, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[476, 503, 562, 488, 519, 493, 519, 549, 516, 557, 515, 556, 503, 498, 517, 540, 503, 517, 505, 503, 536, 447, 483, 506, 546, 485, 500, 508, 509, 514, 484, 468, 501, 527, 518, 520, 551, 495, 544, 559, 508, 525, 504, 529, 531, 474, 488, 510, 479, 504, 527, 553, 518, 497, 506, 478, 510, 508, 490, 517, 516, 508, 506, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[474, 507, 536, 554, 520, 513, 477, 538, 517, 495, 535, 500, 511, 484, 516, 503, 488, 478, 505, 551, 507, 525, 510, 517, 480, 548, 515, 512, 544, 488, 562, 510, 555, 512, 523, 492, 524, 504, 542, 524, 495, 530, 521, 513, 491, 520, 513, 495, 508, 527, 529, 508, 520, 517, 491, 518, 524, 505, 458, 492, 498, 481, 526, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[536, 524, 488, 530, 505, 531, 495, 473, 513, 523, 527, 506, 471, 557, 550, 529, 498, 490, 509, 513, 525, 540, 488, 520, 499, 498, 529, 514, 494, 501, 525, 532, 505, 535, 518, 479, 532, 495, 530, 471, 525, 504, 503, 524, 527, 489, 510, 511, 500, 485, 503, 494, 509, 471, 521, 503, 509, 551, 535, 483, 511, 522, 556, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[483, 485, 530, 535, 513, 477, 495, 486, 501, 544, 516, 513, 518, 493, 502, 511, 534, 514, 563, 496, 494, 499, 509, 503, 521, 490, 523, 492, 523, 540, 492, 519, 507, 545, 524, 506, 489, 538, 532, 516, 517, 528, 521, 511, 542, 515, 555, 518, 498, 516, 491, 509, 494, 496, 486, 538, 515, 499, 515, 491, 524, 509, 521, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 509, 539, 523, 519, 512, 543, 519, 0, 0, 0, 0, 0, 0, 0, 0, 510, 468, 527, 529, 479, 510, 541, 505, 0, 0, 0, 0, 0, 0, 0, 0, 477, 508, 510, 509, 534, 488, 513, 479, 29, 3, 0, 0, 0, 0, 0, 0, 497, 501, 516, 489, 537, 500, 493, 491, 3, 0, 0, 0, 0, 0, 0, 0, 495, 514, 521, 504, 531, 534, 472, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 506, 504, 507, 533, 550, 550, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 503, 503, 493, 544, 514, 475, 16, 10, 0, 0, 0, 0, 0, 0, 498, 502, 544, 498, 532, 523, 531, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4069, 4050, 4027, 4074, 4168, 4085, 4119] avg=4096.0000 max=4176 max_rank=0 min=4027 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[492, 477, 518, 476, 505, 535, 497, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 459, 469, 511, 536, 532, 490, 0, 0, 0, 0, 0, 0, 0, 0, 530, 544, 498, 522, 520, 524, 501, 488, 29, 3, 0, 0, 0, 0, 0, 0, 521, 531, 521, 524, 545, 504, 508, 559, 4, 0, 0, 0, 0, 0, 0, 0, 487, 518, 510, 516, 512, 518, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 519, 476, 529, 497, 497, 570, 533, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 461, 515, 514, 500, 477, 512, 15, 11, 0, 0, 0, 0, 0, 0, 505, 567, 540, 518, 505, 480, 471, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4030, 4159, 4217, 4095, 4130, 4015, 4123] avg=4096.0000 max=4217 max_rank=3 min=3999 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[526, 509, 516, 546, 521, 521, 493, 507, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 465, 515, 532, 517, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 474, 532, 520, 457, 501, 533, 514, 448, 30, 4, 0, 0, 0, 0, 0, 0, 500, 515, 532, 474, 529, 536, 497, 504, 4, 0, 0, 0, 0, 0, 0, 0, 503, 463, 552, 523, 505, 494, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 491, 541, 503, 507, 501, 522, 495, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 538, 558, 519, 520, 494, 521, 490, 17, 11, 0, 0, 0, 0, 0, 0, 495, 487, 545, 522, 524, 521, 514, 555, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4100, 4013, 4091, 4040, 4056, 4166, 4163] avg=4096.0000 max=4166 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[476, 503, 562, 471, 519, 493, 519, 549, 0, 0, 0, 0, 0, 0, 0, 0, 516, 542, 515, 556, 503, 498, 517, 540, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 505, 503, 536, 447, 483, 506, 28, 3, 0, 0, 0, 0, 0, 0, 546, 485, 500, 508, 509, 514, 484, 468, 4, 0, 0, 0, 0, 0, 0, 0, 473, 527, 518, 520, 551, 495, 544, 559, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 504, 529, 531, 474, 488, 510, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 527, 553, 518, 497, 506, 478, 14, 11, 0, 0, 0, 0, 0, 0, 510, 508, 490, 517, 516, 508, 506, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4187, 4031, 4018, 4187, 4069, 4087, 4097] avg=4096.0000 max=4187 max_rank=1 min=4018 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[545, 529, 549, 532, 496, 527, 505, 541, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 510, 514, 514, 526, 486, 511, 0, 0, 0, 0, 0, 0, 0, 0, 504, 535, 527, 539, 496, 484, 506, 508, 31, 4, 0, 0, 0, 0, 0, 0, 541, 519, 508, 475, 494, 491, 507, 510, 4, 0, 0, 0, 0, 0, 0, 0, 524, 543, 514, 527, 478, 502, 468, 509, 0, 0, 0, 0, 0, 0, 0, 0, 508, 482, 524, 481, 480, 489, 523, 473, 0, 0, 0, 0, 0, 0, 0, 0, 523, 543, 524, 516, 522, 510, 506, 522, 16, 11, 0, 0, 0, 0, 0, 0, 482, 486, 508, 502, 494, 537, 534, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4224, 4097, 4134, 4049, 4065, 3960, 4193, 4046] avg=4096.0000 max=4224 max_rank=0 min=3960 min_rank=5 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[474, 507, 536, 533, 520, 513, 477, 538, 0, 0, 0, 0, 0, 0, 0, 0, 517, 482, 535, 500, 511, 484, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 488, 478, 505, 551, 507, 525, 510, 517, 31, 4, 0, 0, 0, 0, 0, 0, 480, 548, 515, 512, 544, 488, 562, 510, 3, 0, 0, 0, 0, 0, 0, 0, 524, 512, 523, 492, 524, 504, 542, 524, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 521, 513, 491, 520, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 529, 508, 520, 517, 491, 518, 17, 10, 0, 0, 0, 0, 0, 0, 524, 505, 458, 492, 498, 481, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4048, 4116, 4162, 4145, 4078, 4145, 3976] avg=4096.0000 max=4162 max_rank=3 min=3976 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[536, 524, 488, 503, 505, 531, 495, 473, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 527, 506, 471, 557, 550, 529, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 509, 513, 525, 540, 488, 520, 32, 7, 0, 0, 0, 0, 0, 0, 499, 498, 529, 514, 494, 501, 525, 532, 6, 2, 0, 0, 0, 0, 0, 0, 473, 535, 518, 479, 532, 495, 530, 471, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 503, 524, 527, 489, 510, 511, 7, 0, 0, 0, 0, 0, 0, 0, 500, 485, 503, 494, 509, 471, 521, 503, 20, 16, 0, 0, 0, 0, 0, 0, 509, 551, 535, 483, 511, 522, 547, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4154, 4122, 4100, 4033, 4100, 4022, 4182] avg=4096.0000 max=4182 max_rank=7 min=4022 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 485, 530, 516, 513, 477, 495, 486, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 516, 513, 518, 493, 502, 511, 0, 0, 0, 0, 0, 0, 0, 0, 534, 514, 563, 496, 494, 499, 509, 503, 28, 3, 0, 0, 0, 0, 0, 0, 521, 490, 523, 492, 523, 540, 492, 519, 4, 0, 0, 0, 0, 0, 0, 0, 479, 545, 524, 506, 489, 538, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 517, 528, 521, 511, 542, 515, 555, 518, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 491, 509, 494, 496, 486, 538, 16, 11, 0, 0, 0, 0, 0, 0, 515, 499, 515, 491, 524, 509, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4083, 4143, 4104, 4129, 4207, 4055, 4062] avg=4096.0000 max=4207 max_rank=5 min=3985 min_rank=0 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4352, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[506, 482, 531, 498, 545, 519, 487, 504, 489, 500, 545, 491, 500, 533, 513, 510, 521, 521, 509, 505, 531, 491, 543, 531, 511, 497, 511, 518, 512, 493, 481, 505, 531, 507, 547, 537, 523, 539, 520, 493, 502, 485, 511, 532, 486, 518, 520, 489, 504, 536, 516, 524, 533, 548, 510, 514, 491, 557, 495, 516, 474, 464, 527, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[532, 498, 535, 473, 511, 535, 492, 525, 532, 526, 518, 545, 476, 476, 493, 527, 501, 459, 491, 509, 555, 507, 473, 509, 496, 509, 496, 513, 508, 508, 514, 507, 510, 536, 556, 513, 480, 509, 514, 527, 513, 535, 487, 542, 539, 500, 492, 496, 487, 522, 495, 539, 515, 546, 527, 508, 523, 526, 468, 548, 500, 521, 515, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[504, 520, 504, 512, 485, 493, 514, 542, 497, 536, 509, 511, 561, 489, 503, 493, 526, 473, 534, 506, 536, 489, 522, 487, 486, 514, 514, 545, 522, 508, 527, 538, 485, 486, 534, 483, 513, 486, 531, 519, 494, 488, 502, 476, 544, 544, 530, 522, 524, 499, 522, 497, 490, 526, 527, 509, 528, 534, 517, 542, 523, 482, 492, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[554, 512, 505, 517, 507, 504, 553, 493, 557, 513, 512, 500, 509, 512, 509, 500, 488, 475, 450, 507, 511, 514, 492, 564, 491, 513, 514, 507, 495, 565, 478, 507, 481, 500, 516, 475, 535, 525, 514, 544, 527, 496, 511, 513, 516, 526, 534, 496, 486, 493, 510, 540, 514, 536, 501, 531, 505, 516, 541, 526, 545, 472, 504, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[488, 512, 518, 562, 540, 518, 504, 523, 539, 515, 488, 490, 492, 512, 503, 527, 546, 491, 503, 500, 526, 509, 533, 546, 535, 526, 552, 531, 494, 565, 527, 475, 529, 499, 528, 466, 527, 511, 493, 508, 539, 485, 504, 484, 478, 499, 512, 532, 476, 488, 493, 499, 527, 527, 519, 519, 479, 503, 492, 488, 513, 502, 509, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[557, 489, 506, 475, 525, 502, 482, 531, 484, 506, 493, 518, 507, 493, 513, 491, 471, 521, 542, 520, 543, 512, 492, 509, 533, 548, 512, 530, 515, 487, 471, 507, 534, 512, 533, 489, 482, 523, 534, 496, 510, 486, 503, 543, 501, 574, 538, 526, 519, 490, 492, 518, 546, 517, 511, 555, 537, 475, 473, 489, 520, 524, 498, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[539, 536, 490, 496, 531, 509, 513, 531, 482, 474, 515, 509, 520, 549, 516, 523, 507, 470, 504, 517, 504, 479, 522, 542, 505, 543, 485, 533, 478, 565, 527, 493, 500, 503, 495, 472, 513, 502, 539, 495, 492, 471, 531, 492, 548, 550, 517, 533, 487, 557, 519, 523, 503, 525, 519, 513, 490, 500, 484, 532, 526, 541, 491, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[529, 518, 483, 470, 528, 505, 523, 500, 546, 514, 536, 495, 525, 523, 513, 519, 485, 507, 466, 516, 515, 545, 498, 557, 523, 492, 488, 526, 512, 513, 523, 509, 543, 497, 490, 553, 471, 518, 516, 513, 517, 483, 508, 514, 522, 506, 494, 522, 510, 501, 488, 489, 514, 547, 475, 525, 503, 547, 538, 480, 499, 533, 532, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 482, 531, 498, 545, 519, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 500, 545, 491, 500, 533, 513, 510, 6, 0, 0, 0, 0, 0, 0, 0, 521, 521, 509, 505, 531, 491, 543, 531, 17, 0, 0, 0, 0, 0, 0, 0, 511, 497, 511, 518, 512, 487, 481, 505, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 547, 537, 523, 539, 520, 493, 1, 0, 0, 0, 0, 0, 0, 0, 502, 485, 511, 532, 486, 517, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 504, 536, 516, 524, 533, 530, 510, 514, 0, 0, 0, 0, 0, 0, 0, 0, 491, 557, 495, 516, 474, 464, 527, 486, 6, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4087, 4169, 4022, 4198, 4042, 4167, 4017] avg=4096.0000 max=4198 max_rank=4 min=4017 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[526, 498, 535, 473, 511, 535, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 518, 545, 476, 476, 493, 527, 6, 0, 0, 0, 0, 0, 0, 0, 501, 459, 491, 509, 555, 507, 473, 509, 17, 0, 0, 0, 0, 0, 0, 0, 496, 509, 496, 513, 508, 501, 514, 507, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 556, 513, 480, 509, 514, 527, 1, 0, 0, 0, 0, 0, 0, 0, 513, 535, 487, 542, 539, 499, 492, 496, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 495, 539, 515, 528, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 468, 548, 500, 521, 515, 530, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4099, 4021, 4044, 4146, 4103, 4121, 4139] avg=4096.0000 max=4146 max_rank=4 min=4021 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 512, 518, 562, 540, 518, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 539, 515, 488, 490, 492, 512, 503, 527, 6, 0, 0, 0, 0, 0, 0, 0, 546, 491, 503, 500, 526, 509, 533, 546, 17, 0, 0, 0, 0, 0, 0, 0, 535, 526, 552, 531, 494, 558, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 529, 499, 528, 466, 527, 511, 493, 508, 1, 0, 0, 0, 0, 0, 0, 0, 539, 485, 504, 484, 478, 498, 512, 532, 0, 0, 0, 0, 0, 0, 0, 0, 476, 488, 493, 499, 527, 509, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 479, 503, 492, 488, 513, 502, 509, 550, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4072, 4171, 4198, 4062, 4032, 4030, 4044] avg=4096.0000 max=4198 max_rank=3 min=4030 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 520, 504, 512, 485, 493, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 497, 536, 509, 511, 561, 489, 503, 493, 6, 0, 0, 0, 0, 0, 0, 0, 526, 473, 534, 506, 536, 489, 522, 487, 17, 0, 0, 0, 0, 0, 0, 0, 486, 514, 514, 545, 522, 501, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 485, 486, 534, 483, 513, 486, 531, 519, 1, 0, 0, 0, 0, 0, 0, 0, 494, 488, 502, 476, 544, 543, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 522, 497, 490, 508, 527, 509, 0, 0, 0, 0, 0, 0, 0, 0, 528, 534, 517, 542, 523, 482, 492, 519, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4105, 4090, 4147, 4038, 4099, 4076, 4145] avg=4096.0000 max=4147 max_rank=3 min=4038 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[548, 512, 505, 517, 507, 504, 553, 493, 0, 0, 0, 0, 0, 0, 0, 0, 557, 513, 512, 500, 509, 512, 509, 500, 6, 0, 0, 0, 0, 0, 0, 0, 488, 475, 450, 507, 511, 514, 492, 564, 17, 0, 0, 0, 0, 0, 0, 0, 491, 513, 514, 507, 495, 558, 478, 507, 0, 0, 0, 0, 0, 0, 0, 0, 481, 500, 516, 475, 535, 525, 514, 544, 1, 0, 0, 0, 0, 0, 0, 0, 527, 496, 511, 513, 516, 525, 534, 496, 0, 0, 0, 0, 0, 0, 0, 0, 486, 493, 510, 540, 514, 518, 501, 531, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 541, 526, 545, 472, 504, 511, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4118, 4018, 4063, 4091, 4118, 4093, 4128] avg=4096.0000 max=4139 max_rank=0 min=4018 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[550, 489, 506, 475, 525, 502, 482, 531, 0, 0, 0, 0, 0, 0, 0, 0, 484, 506, 493, 518, 507, 493, 513, 491, 7, 0, 0, 0, 0, 0, 0, 0, 471, 521, 542, 520, 543, 512, 492, 509, 16, 0, 0, 0, 0, 0, 0, 0, 533, 548, 512, 530, 515, 481, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 534, 512, 533, 489, 482, 523, 534, 496, 1, 0, 0, 0, 0, 0, 0, 0, 510, 486, 503, 543, 501, 573, 538, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 490, 492, 518, 546, 500, 511, 555, 0, 0, 0, 0, 0, 0, 0, 0, 537, 475, 473, 489, 520, 524, 498, 535, 6, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4012, 4126, 4097, 4104, 4180, 4131, 4058] avg=4096.0000 max=4180 max_rank=5 min=4012 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[533, 536, 490, 496, 531, 509, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 482, 474, 515, 509, 520, 549, 516, 523, 6, 0, 0, 0, 0, 0, 0, 0, 507, 470, 504, 517, 504, 479, 522, 542, 17, 0, 0, 0, 0, 0, 0, 0, 505, 543, 485, 533, 478, 558, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 495, 472, 513, 502, 539, 495, 1, 0, 0, 0, 0, 0, 0, 0, 492, 471, 531, 492, 548, 549, 517, 533, 0, 0, 0, 0, 0, 0, 0, 0, 487, 557, 519, 523, 503, 507, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 484, 532, 526, 541, 491, 498, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4094, 4062, 4122, 4020, 4133, 4128, 4070] avg=4096.0000 max=4139 max_rank=0 min=4020 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 518, 483, 470, 528, 505, 523, 500, 0, 0, 0, 0, 0, 0, 0, 0, 546, 514, 536, 495, 525, 523, 513, 519, 10, 0, 0, 0, 0, 0, 0, 0, 485, 507, 466, 516, 515, 545, 498, 557, 22, 0, 0, 0, 0, 0, 0, 0, 523, 492, 488, 526, 512, 502, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 543, 497, 490, 553, 471, 518, 516, 513, 5, 3, 0, 0, 0, 0, 0, 0, 517, 483, 508, 514, 522, 501, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 488, 489, 514, 520, 475, 525, 0, 0, 0, 0, 0, 0, 0, 0, 503, 547, 538, 480, 499, 533, 532, 518, 11, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4181, 4111, 4075, 4109, 4061, 4022, 4167] avg=4096.0000 max=4181 max_rank=1 min=4022 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4096, 4096, 4352, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[536, 522, 527, 495, 522, 543, 516, 529, 484, 531, 499, 508, 529, 519, 522, 498, 503, 505, 530, 527, 555, 527, 482, 507, 535, 486, 530, 525, 489, 462, 480, 523, 535, 530, 476, 497, 511, 474, 534, 508, 531, 474, 488, 521, 511, 469, 513, 528, 548, 486, 484, 521, 501, 518, 519, 494, 509, 527, 541, 528, 497, 517, 513, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[509, 499, 509, 496, 520, 507, 519, 489, 483, 503, 502, 477, 521, 550, 523, 543, 535, 503, 535, 465, 490, 528, 483, 505, 518, 521, 514, 513, 536, 487, 518, 515, 523, 526, 532, 482, 506, 492, 499, 524, 516, 537, 527, 551, 513, 506, 514, 522, 464, 550, 491, 510, 470, 533, 497, 504, 524, 513, 533, 497, 549, 506, 529, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[476, 541, 490, 530, 523, 529, 506, 554, 552, 503, 524, 496, 497, 487, 497, 530, 496, 515, 536, 496, 556, 527, 467, 518, 500, 483, 519, 519, 514, 517, 511, 501, 505, 524, 540, 500, 504, 551, 528, 526, 523, 485, 529, 544, 532, 492, 466, 497, 479, 536, 513, 499, 500, 528, 542, 549, 492, 494, 479, 507, 459, 510, 492, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[514, 535, 522, 515, 497, 506, 508, 500, 503, 497, 483, 500, 518, 495, 551, 477, 487, 501, 507, 543, 504, 497, 512, 475, 459, 519, 507, 493, 490, 528, 551, 484, 529, 509, 510, 522, 487, 490, 536, 515, 544, 535, 509, 526, 515, 508, 561, 506, 539, 511, 500, 507, 532, 533, 515, 538, 491, 550, 514, 503, 524, 499, 541, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[476, 504, 500, 500, 512, 518, 490, 501, 509, 512, 531, 486, 553, 476, 509, 497, 519, 512, 535, 546, 505, 518, 500, 512, 513, 518, 528, 573, 545, 463, 508, 547, 531, 481, 528, 506, 496, 508, 514, 545, 498, 503, 514, 515, 485, 494, 514, 520, 525, 513, 487, 490, 533, 466, 533, 496, 555, 500, 497, 550, 510, 510, 500, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[518, 501, 492, 530, 546, 489, 512, 511, 517, 548, 508, 525, 526, 518, 535, 534, 488, 517, 558, 478, 521, 506, 497, 524, 492, 477, 491, 516, 561, 470, 504, 505, 495, 545, 510, 499, 512, 484, 476, 486, 514, 534, 506, 509, 534, 502, 520, 532, 526, 513, 510, 469, 507, 493, 515, 521, 492, 536, 491, 531, 518, 521, 519, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[520, 524, 522, 512, 458, 556, 522, 517, 502, 498, 518, 503, 537, 508, 500, 513, 512, 518, 512, 537, 511, 491, 551, 512, 538, 526, 536, 471, 520, 485, 522, 484, 503, 514, 525, 466, 519, 515, 494, 487, 536, 469, 531, 518, 493, 511, 496, 561, 500, 545, 493, 534, 508, 533, 507, 532, 473, 505, 509, 495, 534, 496, 515, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[520, 504, 485, 528, 485, 515, 533, 509, 481, 523, 478, 518, 535, 482, 521, 478, 480, 547, 509, 505, 500, 508, 540, 510, 489, 471, 511, 549, 513, 517, 513, 549, 503, 521, 499, 483, 509, 516, 537, 491, 508, 512, 532, 478, 537, 525, 511, 520, 531, 523, 521, 535, 532, 501, 540, 497, 518, 490, 512, 527, 535, 494, 523, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 501, 492, 530, 546, 486, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 548, 508, 525, 526, 518, 535, 534, 4, 3, 3, 0, 0, 0, 0, 0, 488, 517, 555, 478, 521, 506, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 492, 477, 491, 516, 561, 470, 504, 505, 8, 4, 1, 0, 0, 0, 0, 0, 495, 545, 510, 499, 512, 484, 476, 486, 18, 0, 0, 0, 0, 0, 0, 0, 514, 534, 506, 509, 534, 502, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 510, 469, 507, 493, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 492, 536, 491, 531, 518, 521, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4221, 4086, 4029, 4025, 4132, 4046, 4133] avg=4096.0000 max=4221 max_rank=1 min=4025 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 499, 509, 496, 520, 504, 519, 489, 0, 0, 0, 0, 0, 0, 0, 0, 483, 503, 502, 477, 521, 550, 523, 543, 4, 3, 3, 0, 0, 0, 0, 0, 535, 503, 532, 465, 490, 528, 483, 505, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 514, 513, 536, 487, 518, 515, 9, 3, 1, 0, 0, 0, 0, 0, 523, 526, 532, 482, 506, 492, 499, 524, 18, 0, 0, 0, 0, 0, 0, 0, 516, 537, 527, 551, 513, 506, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 464, 541, 491, 510, 470, 533, 497, 504, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 533, 497, 549, 506, 529, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4112, 4041, 4135, 4102, 4167, 4010, 4156] avg=4096.0000 max=4167 max_rank=5 min=4010 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[476, 541, 490, 530, 523, 526, 506, 554, 0, 0, 0, 0, 0, 0, 0, 0, 552, 503, 524, 496, 497, 487, 497, 530, 4, 3, 3, 0, 0, 0, 0, 0, 496, 515, 533, 496, 556, 527, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 500, 483, 519, 519, 514, 517, 511, 501, 9, 4, 1, 0, 0, 0, 0, 0, 505, 524, 540, 500, 504, 551, 528, 526, 17, 0, 0, 0, 0, 0, 0, 0, 523, 485, 529, 544, 532, 492, 466, 479, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 513, 499, 500, 528, 542, 549, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 479, 507, 459, 510, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4096, 4108, 4078, 4195, 4050, 4137, 3958] avg=4096.0000 max=4195 max_rank=4 min=3958 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 535, 522, 515, 497, 503, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 483, 500, 518, 495, 551, 477, 3, 3, 3, 0, 0, 0, 0, 0, 487, 501, 504, 543, 504, 497, 512, 475, 0, 0, 0, 0, 0, 0, 0, 0, 459, 519, 507, 493, 490, 528, 551, 484, 8, 3, 1, 0, 0, 0, 0, 0, 529, 509, 510, 522, 487, 490, 536, 515, 17, 0, 0, 0, 0, 0, 0, 0, 544, 535, 509, 526, 515, 508, 561, 488, 0, 0, 0, 0, 0, 0, 0, 0, 539, 503, 500, 507, 532, 533, 515, 538, 0, 0, 0, 0, 0, 0, 0, 0, 491, 550, 514, 503, 524, 499, 541, 485, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4033, 4023, 4043, 4115, 4186, 4167, 4107] avg=4096.0000 max=4186 max_rank=5 min=4023 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 524, 522, 512, 458, 551, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 518, 503, 537, 508, 500, 513, 7, 7, 5, 0, 0, 0, 0, 0, 512, 518, 505, 537, 511, 491, 551, 512, 0, 0, 0, 0, 0, 0, 0, 0, 538, 526, 536, 471, 520, 485, 522, 484, 13, 7, 5, 0, 0, 0, 0, 0, 503, 514, 525, 466, 519, 515, 494, 487, 21, 0, 0, 0, 0, 0, 0, 0, 536, 469, 531, 518, 493, 511, 496, 535, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 493, 534, 508, 533, 507, 532, 0, 0, 0, 0, 0, 0, 0, 0, 473, 505, 509, 495, 534, 496, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4098, 4137, 4107, 4044, 4089, 4139, 4028] avg=4096.0000 max=4139 max_rank=6 min=4028 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[476, 504, 500, 500, 512, 515, 490, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 531, 486, 553, 476, 509, 497, 4, 3, 3, 0, 0, 0, 0, 0, 519, 512, 532, 546, 505, 518, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 528, 573, 545, 463, 508, 547, 8, 4, 1, 0, 0, 0, 0, 0, 531, 481, 528, 506, 496, 508, 514, 545, 18, 0, 0, 0, 0, 0, 0, 0, 498, 503, 514, 515, 485, 494, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 487, 490, 533, 466, 533, 496, 0, 0, 0, 0, 0, 0, 0, 0, 555, 500, 497, 550, 510, 510, 500, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4083, 4144, 4208, 4127, 4024, 4035, 4149] avg=4096.0000 max=4208 max_rank=3 min=3998 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 504, 485, 528, 485, 512, 533, 509, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 478, 518, 535, 482, 521, 478, 3, 3, 3, 0, 0, 0, 0, 0, 480, 547, 506, 505, 500, 508, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 489, 471, 511, 549, 513, 517, 513, 549, 9, 3, 1, 0, 0, 0, 0, 0, 503, 521, 499, 483, 509, 516, 537, 491, 18, 0, 0, 0, 0, 0, 0, 0, 508, 512, 532, 478, 537, 525, 511, 501, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 521, 535, 532, 501, 540, 497, 0, 0, 0, 0, 0, 0, 0, 0, 518, 490, 512, 527, 535, 494, 523, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4025, 4096, 4125, 4077, 4104, 4171, 4094] avg=4096.0000 max=4171 max_rank=6 min=4025 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[536, 522, 527, 495, 522, 540, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 484, 531, 499, 508, 529, 519, 522, 498, 4, 3, 3, 0, 0, 0, 0, 0, 503, 505, 527, 527, 555, 527, 482, 507, 0, 0, 0, 0, 0, 0, 0, 0, 535, 486, 530, 525, 489, 462, 480, 523, 8, 4, 1, 0, 0, 0, 0, 0, 535, 530, 476, 497, 511, 474, 534, 508, 18, 0, 0, 0, 0, 0, 0, 0, 531, 474, 488, 521, 511, 469, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 548, 478, 484, 521, 501, 518, 519, 494, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 541, 528, 497, 517, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4100, 4133, 4043, 4083, 4016, 4063, 4143] avg=4096.0000 max=4187 max_rank=0 min=4016 min_rank=5 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[526, 513, 516, 526, 503, 485, 501, 501, 531, 512, 503, 577, 502, 527, 502, 514, 524, 514, 523, 491, 521, 502, 527, 493, 470, 497, 510, 543, 529, 528, 519, 548, 499, 517, 544, 514, 525, 467, 494, 505, 526, 491, 478, 534, 471, 513, 473, 502, 516, 512, 502, 494, 525, 514, 517, 525, 555, 505, 505, 507, 516, 533, 510, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[509, 479, 500, 513, 487, 526, 558, 507, 480, 496, 469, 525, 520, 514, 501, 505, 503, 545, 513, 491, 526, 517, 527, 498, 491, 522, 528, 509, 514, 540, 540, 520, 519, 495, 516, 522, 490, 518, 575, 562, 471, 531, 506, 500, 510, 521, 523, 495, 505, 539, 500, 506, 511, 504, 514, 495, 533, 509, 492, 499, 495, 514, 491, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[539, 507, 518, 511, 490, 491, 506, 493, 511, 507, 490, 490, 543, 504, 514, 545, 494, 518, 493, 503, 449, 515, 514, 527, 554, 554, 478, 486, 509, 486, 486, 518, 491, 482, 532, 544, 552, 531, 509, 549, 511, 575, 525, 511, 493, 534, 521, 513, 503, 506, 475, 468, 538, 503, 537, 488, 522, 523, 520, 502, 562, 514, 491, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[468, 476, 509, 504, 490, 480, 555, 493, 502, 501, 522, 518, 478, 534, 499, 523, 502, 520, 480, 508, 501, 489, 497, 549, 536, 513, 494, 479, 513, 511, 532, 504, 512, 528, 545, 449, 533, 502, 483, 530, 501, 536, 519, 481, 507, 505, 537, 486, 543, 500, 546, 540, 530, 554, 507, 528, 529, 537, 537, 505, 537, 511, 536, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[514, 529, 516, 513, 538, 498, 514, 532, 518, 539, 505, 511, 527, 485, 515, 522, 541, 456, 489, 474, 499, 545, 487, 529, 489, 533, 523, 498, 517, 530, 512, 486, 531, 484, 517, 522, 528, 489, 526, 490, 545, 552, 521, 492, 516, 483, 519, 504, 503, 534, 475, 545, 521, 482, 514, 503, 511, 511, 512, 487, 494, 535, 528, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[491, 518, 540, 526, 530, 524, 521, 500, 507, 500, 528, 536, 494, 558, 528, 489, 531, 556, 535, 472, 530, 492, 529, 538, 504, 539, 524, 515, 530, 538, 529, 488, 503, 444, 493, 527, 525, 489, 547, 505, 535, 470, 503, 485, 540, 476, 466, 515, 474, 535, 507, 472, 547, 481, 496, 471, 505, 537, 477, 490, 517, 525, 538, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[531, 484, 506, 514, 499, 514, 481, 517, 512, 514, 512, 522, 500, 492, 538, 511, 500, 522, 469, 468, 509, 504, 536, 526, 525, 532, 520, 513, 498, 540, 546, 483, 484, 509, 518, 468, 540, 510, 538, 480, 513, 500, 490, 511, 507, 498, 511, 548, 511, 557, 474, 534, 532, 498, 549, 512, 526, 502, 486, 523, 513, 526, 535, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[497, 506, 504, 504, 490, 510, 512, 475, 476, 552, 507, 572, 525, 529, 511, 510, 526, 521, 532, 510, 500, 478, 526, 502, 516, 530, 507, 477, 529, 515, 509, 477, 546, 520, 476, 505, 525, 497, 523, 524, 511, 522, 547, 508, 513, 506, 522, 511, 532, 507, 526, 481, 511, 579, 498, 463, 531, 474, 475, 487, 530, 544, 537, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 479, 500, 513, 487, 526, 558, 507, 22, 7, 0, 0, 0, 0, 0, 0, 480, 496, 469, 509, 520, 514, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 513, 491, 526, 517, 527, 498, 12, 7, 0, 0, 0, 0, 0, 0, 491, 503, 528, 509, 514, 540, 540, 520, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 516, 522, 485, 518, 575, 562, 0, 0, 0, 0, 0, 0, 0, 0, 471, 531, 506, 500, 510, 521, 523, 495, 9, 5, 0, 0, 0, 0, 0, 0, 505, 539, 500, 506, 511, 504, 514, 495, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 492, 499, 495, 514, 491, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 3994, 4139, 4145, 4192, 4071, 4074, 4045] avg=4096.0000 max=4192 max_rank=4 min=3994 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[539, 507, 518, 511, 490, 491, 506, 493, 22, 8, 0, 0, 0, 0, 0, 0, 511, 507, 490, 476, 543, 504, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 493, 503, 449, 515, 514, 527, 13, 6, 0, 0, 0, 0, 0, 0, 554, 533, 478, 486, 509, 486, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 532, 544, 546, 531, 509, 549, 0, 0, 0, 0, 0, 0, 0, 0, 511, 575, 525, 511, 493, 534, 521, 513, 8, 6, 0, 0, 0, 0, 0, 0, 503, 506, 475, 468, 538, 503, 537, 488, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 520, 502, 562, 514, 491, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4090, 4032, 4050, 4184, 4197, 4018, 4112] avg=4096.0000 max=4197 max_rank=5 min=4018 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 513, 516, 526, 503, 485, 501, 501, 23, 7, 0, 0, 0, 0, 0, 0, 531, 512, 503, 560, 502, 527, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 523, 491, 521, 502, 527, 493, 12, 7, 0, 0, 0, 0, 0, 0, 470, 478, 510, 543, 529, 528, 519, 548, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 544, 514, 519, 467, 494, 505, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 478, 534, 471, 513, 473, 502, 10, 6, 0, 0, 0, 0, 0, 0, 516, 512, 502, 494, 525, 514, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 532, 505, 505, 507, 516, 533, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4151, 4114, 4125, 4059, 4004, 4105, 4109] avg=4096.0000 max=4151 max_rank=1 min=4004 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[468, 476, 509, 504, 490, 480, 555, 493, 22, 7, 0, 0, 0, 0, 0, 0, 502, 501, 522, 502, 478, 534, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 480, 508, 501, 489, 497, 549, 12, 7, 0, 0, 0, 0, 0, 0, 536, 494, 494, 479, 513, 511, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 545, 449, 527, 502, 483, 530, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 519, 481, 507, 505, 537, 486, 9, 6, 0, 0, 0, 0, 0, 0, 543, 500, 546, 540, 530, 554, 507, 528, 0, 0, 0, 0, 0, 0, 0, 0, 507, 537, 537, 505, 537, 511, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4061, 4065, 4063, 4076, 4087, 4248, 4164] avg=4096.0000 max=4248 max_rank=6 min=4004 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 529, 516, 513, 538, 498, 514, 532, 21, 7, 0, 0, 0, 0, 0, 0, 518, 539, 505, 496, 527, 485, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 541, 456, 489, 474, 499, 545, 487, 529, 13, 6, 0, 0, 0, 0, 0, 0, 489, 513, 523, 498, 517, 530, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0, 531, 484, 517, 522, 522, 489, 526, 490, 0, 0, 0, 0, 0, 0, 0, 0, 545, 552, 521, 492, 516, 483, 519, 504, 9, 6, 0, 0, 0, 0, 0, 0, 503, 534, 475, 545, 521, 482, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 490, 511, 512, 487, 494, 535, 528, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4107, 4039, 4068, 4081, 4147, 4077, 4067] avg=4096.0000 max=4182 max_rank=0 min=4039 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[531, 484, 506, 514, 499, 514, 481, 517, 22, 7, 0, 0, 0, 0, 0, 0, 512, 514, 512, 506, 500, 492, 538, 511, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 469, 468, 509, 504, 536, 526, 13, 7, 0, 0, 0, 0, 0, 0, 525, 512, 520, 513, 498, 540, 546, 483, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 518, 468, 534, 510, 538, 480, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 490, 511, 507, 498, 511, 548, 9, 6, 0, 0, 0, 0, 0, 0, 511, 557, 474, 534, 532, 498, 549, 512, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 486, 523, 513, 526, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4085, 4054, 4137, 4041, 4093, 4167, 4116] avg=4096.0000 max=4167 max_rank=6 min=4041 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 518, 540, 526, 530, 524, 521, 500, 25, 11, 0, 0, 0, 0, 0, 0, 507, 500, 528, 511, 494, 558, 528, 489, 0, 0, 0, 0, 0, 0, 0, 0, 531, 556, 535, 472, 530, 492, 529, 538, 16, 11, 0, 0, 0, 0, 0, 0, 504, 512, 524, 515, 530, 538, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 503, 444, 493, 527, 517, 489, 547, 505, 0, 0, 0, 0, 0, 0, 0, 0, 535, 470, 503, 485, 540, 476, 466, 515, 14, 8, 1, 0, 0, 0, 0, 0, 474, 535, 507, 472, 546, 481, 496, 471, 0, 0, 0, 0, 0, 0, 0, 0, 480, 537, 477, 490, 517, 525, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4115, 4210, 4140, 4025, 4013, 3982, 4097] avg=4096.0000 max=4210 max_rank=2 min=3982 min_rank=6 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16117 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[497, 506, 504, 504, 490, 510, 512, 475, 22, 7, 0, 0, 0, 0, 0, 0, 476, 552, 507, 555, 525, 529, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 532, 510, 500, 478, 526, 502, 13, 7, 0, 0, 0, 0, 0, 0, 516, 510, 507, 477, 529, 515, 509, 477, 0, 0, 0, 0, 0, 0, 0, 0, 546, 520, 476, 505, 519, 497, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 547, 508, 513, 506, 522, 511, 10, 6, 0, 0, 0, 0, 0, 0, 532, 507, 526, 481, 511, 579, 498, 463, 0, 0, 0, 0, 0, 0, 0, 0, 509, 474, 475, 487, 530, 544, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4165, 4115, 4040, 4110, 4156, 4097, 4058] avg=4096.0000 max=4165 max_rank=1 min=4027 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[485, 492, 555, 509, 518, 505, 465, 520, 519, 557, 537, 492, 493, 510, 481, 550, 533, 527, 523, 503, 510, 522, 519, 496, 498, 483, 537, 534, 497, 514, 501, 534, 509, 477, 507, 516, 509, 511, 522, 519, 546, 475, 524, 512, 495, 519, 520, 501, 480, 517, 516, 489, 508, 563, 498, 493, 480, 547, 481, 517, 517, 522, 542, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[498, 539, 504, 492, 516, 479, 530, 526, 508, 500, 512, 499, 512, 486, 506, 494, 546, 511, 518, 542, 516, 525, 504, 480, 510, 513, 515, 502, 507, 519, 519, 500, 513, 526, 494, 461, 482, 514, 525, 456, 531, 521, 512, 558, 564, 477, 524, 532, 498, 527, 517, 534, 533, 530, 462, 498, 481, 511, 536, 523, 532, 547, 505, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[512, 519, 511, 501, 546, 490, 537, 494, 514, 520, 536, 552, 503, 517, 508, 474, 511, 478, 504, 501, 524, 528, 500, 512, 554, 528, 506, 494, 473, 470, 518, 480, 514, 531, 513, 511, 525, 488, 527, 509, 516, 552, 483, 522, 512, 519, 520, 517, 511, 500, 512, 516, 510, 518, 513, 490, 507, 462, 559, 518, 516, 522, 519, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[494, 537, 510, 491, 465, 490, 487, 510, 531, 503, 550, 530, 527, 514, 475, 526, 507, 517, 563, 521, 559, 438, 528, 513, 523, 518, 486, 494, 494, 530, 517, 522, 515, 511, 507, 485, 517, 540, 499, 504, 535, 497, 502, 505, 486, 489, 479, 512, 515, 498, 560, 535, 526, 571, 527, 528, 498, 517, 490, 503, 519, 504, 521, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[454, 495, 550, 509, 531, 530, 488, 523, 502, 480, 511, 498, 508, 548, 515, 505, 490, 511, 514, 527, 524, 498, 544, 558, 500, 515, 523, 496, 527, 515, 503, 497, 503, 497, 517, 520, 505, 494, 489, 494, 505, 492, 514, 510, 518, 499, 541, 484, 490, 534, 510, 531, 521, 558, 519, 538, 528, 511, 494, 526, 530, 502, 490, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[527, 515, 512, 518, 474, 506, 518, 502, 562, 516, 483, 544, 518, 514, 493, 515, 511, 476, 505, 502, 534, 499, 522, 549, 530, 530, 524, 513, 498, 498, 559, 564, 480, 526, 515, 522, 525, 527, 521, 484, 465, 484, 505, 506, 510, 507, 500, 517, 503, 507, 524, 482, 539, 542, 498, 470, 511, 514, 523, 508, 522, 488, 514, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[495, 528, 498, 527, 510, 525, 529, 508, 476, 487, 525, 500, 551, 528, 511, 523, 521, 509, 512, 542, 492, 511, 561, 511, 509, 525, 532, 510, 517, 514, 535, 494, 495, 482, 517, 509, 541, 498, 524, 523, 498, 475, 524, 503, 478, 488, 519, 506, 546, 533, 474, 512, 550, 495, 481, 531, 478, 523, 497, 512, 516, 488, 540, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[524, 485, 533, 466, 540, 499, 493, 519, 504, 515, 520, 512, 513, 518, 491, 478, 502, 519, 516, 510, 535, 541, 519, 528, 507, 487, 496, 517, 559, 540, 530, 497, 499, 525, 496, 517, 489, 515, 502, 509, 544, 486, 521, 528, 520, 504, 447, 537, 527, 538, 504, 530, 526, 545, 479, 528, 513, 496, 511, 512, 516, 474, 511, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[498, 539, 504, 492, 516, 479, 530, 526, 21, 3, 0, 0, 0, 0, 0, 0, 508, 500, 509, 499, 512, 486, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 546, 511, 518, 542, 516, 525, 469, 480, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 515, 502, 507, 519, 510, 500, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 494, 461, 482, 514, 525, 456, 32, 0, 0, 0, 0, 0, 0, 0, 531, 521, 512, 558, 564, 477, 524, 532, 13, 7, 0, 0, 0, 0, 0, 0, 498, 527, 517, 534, 533, 496, 462, 498, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 536, 523, 532, 547, 505, 516, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4014, 4107, 4076, 4003, 4239, 4065, 4156] avg=4096.0000 max=4239 max_rank=5 min=4003 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[485, 492, 555, 509, 518, 505, 465, 520, 23, 4, 0, 0, 0, 0, 0, 0, 519, 557, 533, 492, 493, 510, 481, 550, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 523, 503, 510, 522, 482, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 483, 537, 534, 497, 514, 492, 534, 0, 0, 0, 0, 0, 0, 0, 0, 509, 477, 507, 516, 509, 511, 522, 519, 33, 0, 0, 0, 0, 0, 0, 0, 546, 475, 524, 512, 495, 519, 520, 501, 14, 7, 0, 0, 0, 0, 0, 0, 480, 517, 516, 489, 508, 526, 498, 493, 0, 0, 0, 0, 0, 0, 0, 0, 480, 547, 481, 517, 517, 522, 542, 517, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4135, 4096, 4089, 4103, 4113, 4027, 4129] avg=4096.0000 max=4135 max_rank=1 min=4027 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 519, 511, 501, 546, 490, 537, 494, 21, 3, 0, 0, 0, 0, 0, 0, 514, 520, 532, 552, 503, 517, 508, 474, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 504, 501, 524, 528, 465, 512, 0, 0, 0, 0, 0, 0, 0, 0, 554, 528, 506, 494, 473, 470, 509, 480, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 513, 511, 525, 488, 527, 509, 32, 0, 0, 0, 0, 0, 0, 0, 516, 552, 483, 522, 512, 519, 520, 517, 13, 7, 0, 0, 0, 0, 0, 0, 511, 500, 512, 516, 510, 484, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 507, 462, 559, 518, 516, 522, 519, 521, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4120, 4023, 4014, 4150, 4161, 4036, 4130] avg=4096.0000 max=4161 max_rank=5 min=4014 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 528, 498, 527, 510, 525, 529, 508, 20, 4, 0, 0, 0, 0, 0, 0, 476, 487, 521, 500, 551, 528, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 521, 509, 512, 542, 492, 511, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 532, 510, 517, 514, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 495, 482, 517, 509, 541, 498, 524, 523, 36, 0, 0, 0, 0, 0, 0, 0, 498, 475, 524, 503, 478, 488, 519, 506, 12, 8, 0, 0, 0, 0, 0, 0, 546, 533, 474, 512, 550, 463, 481, 531, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 497, 512, 516, 488, 540, 496, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4097, 4119, 4126, 4125, 4011, 4090, 4056] avg=4096.0000 max=4144 max_rank=0 min=4011 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[454, 495, 550, 509, 531, 530, 488, 523, 22, 4, 0, 0, 0, 0, 0, 0, 502, 480, 508, 498, 508, 548, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 490, 511, 514, 527, 524, 498, 505, 558, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 523, 496, 527, 515, 494, 497, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 517, 520, 505, 494, 489, 494, 35, 0, 0, 0, 0, 0, 0, 0, 505, 492, 514, 510, 518, 499, 541, 484, 14, 7, 0, 0, 0, 0, 0, 0, 490, 534, 510, 531, 521, 522, 519, 538, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 494, 526, 530, 502, 490, 515, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4064, 4127, 4067, 4054, 4084, 4165, 4101] avg=4096.0000 max=4165 max_rank=6 min=4054 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[494, 537, 510, 491, 465, 490, 487, 510, 23, 4, 0, 0, 0, 0, 0, 0, 531, 503, 546, 530, 527, 514, 475, 526, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 563, 521, 559, 438, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 523, 518, 486, 494, 494, 530, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 507, 485, 517, 540, 499, 504, 34, 0, 0, 0, 0, 0, 0, 0, 535, 497, 502, 505, 486, 489, 479, 512, 14, 7, 0, 0, 0, 0, 0, 0, 515, 498, 560, 535, 526, 534, 527, 528, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 490, 503, 519, 504, 521, 503, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4152, 4108, 4075, 4112, 4026, 4223, 4061] avg=4096.0000 max=4223 max_rank=6 min=4011 min_rank=0 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 515, 512, 518, 474, 506, 518, 502, 25, 7, 0, 0, 0, 0, 0, 0, 562, 516, 476, 544, 518, 514, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 511, 476, 505, 502, 534, 499, 479, 549, 0, 0, 0, 0, 0, 0, 0, 0, 530, 530, 524, 513, 498, 498, 540, 564, 0, 0, 0, 0, 0, 0, 0, 0, 480, 526, 515, 522, 525, 527, 521, 484, 36, 0, 0, 0, 0, 0, 0, 0, 465, 484, 505, 506, 510, 507, 500, 517, 19, 13, 0, 0, 0, 0, 0, 0, 503, 507, 524, 482, 539, 498, 498, 470, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 523, 508, 522, 488, 514, 498, 7, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4138, 4055, 4197, 4136, 4026, 4021, 4091] avg=4096.0000 max=4197 max_rank=3 min=4021 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 485, 533, 466, 540, 499, 493, 519, 22, 4, 0, 0, 0, 0, 0, 0, 504, 515, 517, 512, 513, 518, 491, 478, 0, 0, 0, 0, 0, 0, 0, 0, 502, 519, 516, 510, 535, 541, 482, 528, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 496, 517, 559, 540, 521, 497, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 496, 517, 489, 515, 502, 509, 33, 0, 0, 0, 0, 0, 0, 0, 544, 486, 521, 528, 520, 504, 447, 537, 14, 7, 0, 0, 0, 0, 0, 0, 527, 538, 504, 530, 526, 509, 479, 528, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 511, 512, 516, 474, 511, 506, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4048, 4133, 4124, 4085, 4108, 4141, 4044] avg=4096.0000 max=4141 max_rank=6 min=4044 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[520, 494, 495, 514, 501, 505, 522, 537, 498, 495, 506, 528, 525, 501, 485, 512, 508, 528, 503, 507, 564, 520, 523, 498, 485, 504, 495, 533, 501, 493, 524, 527, 525, 527, 500, 517, 503, 519, 521, 514, 512, 527, 550, 545, 492, 506, 489, 522, 488, 517, 504, 528, 496, 490, 541, 488, 524, 520, 480, 541, 524, 520, 480, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[511, 530, 507, 495, 497, 500, 522, 477, 500, 515, 496, 501, 506, 493, 511, 509, 533, 525, 506, 499, 553, 502, 480, 490, 509, 538, 548, 543, 517, 503, 534, 514, 536, 506, 487, 533, 548, 499, 542, 525, 496, 501, 508, 532, 489, 544, 538, 508, 529, 508, 511, 508, 504, 491, 496, 495, 496, 496, 504, 534, 513, 508, 518, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[511, 578, 501, 517, 516, 491, 510, 527, 477, 513, 540, 504, 547, 475, 549, 524, 519, 527, 477, 526, 494, 512, 480, 508, 516, 488, 522, 549, 494, 521, 513, 497, 485, 490, 527, 505, 538, 476, 526, 501, 500, 525, 489, 537, 516, 509, 504, 493, 517, 472, 539, 509, 528, 538, 516, 536, 510, 510, 519, 512, 472, 491, 510, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[511, 527, 530, 517, 493, 457, 517, 526, 475, 488, 521, 561, 523, 495, 515, 542, 521, 504, 484, 524, 506, 512, 524, 488, 525, 525, 572, 501, 520, 513, 542, 455, 511, 487, 473, 509, 510, 529, 502, 512, 505, 523, 523, 503, 519, 496, 530, 499, 504, 484, 530, 516, 517, 533, 524, 535, 505, 486, 500, 492, 511, 556, 515, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[525, 546, 510, 504, 487, 498, 494, 504, 500, 535, 537, 502, 526, 509, 523, 477, 525, 500, 504, 509, 502, 527, 495, 533, 523, 485, 483, 539, 522, 556, 488, 492, 513, 529, 490, 497, 524, 521, 457, 509, 527, 475, 513, 503, 488, 496, 556, 521, 534, 504, 530, 507, 490, 545, 538, 520, 488, 503, 558, 512, 531, 494, 513, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[512, 515, 526, 477, 521, 521, 492, 513, 550, 494, 528, 506, 553, 517, 474, 473, 532, 497, 537, 496, 493, 515, 493, 481, 491, 491, 548, 488, 540, 531, 521, 514, 511, 519, 477, 503, 531, 478, 575, 558, 522, 499, 514, 495, 546, 525, 538, 474, 496, 539, 487, 493, 494, 499, 518, 474, 539, 498, 525, 534, 533, 499, 517, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[509, 552, 541, 534, 525, 545, 492, 517, 511, 499, 516, 525, 512, 480, 537, 515, 496, 492, 504, 527, 492, 495, 529, 498, 508, 520, 513, 499, 507, 520, 545, 514, 516, 475, 524, 523, 464, 515, 531, 512, 477, 505, 502, 513, 543, 537, 525, 519, 508, 500, 512, 473, 501, 471, 522, 509, 459, 493, 532, 539, 511, 513, 573, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[548, 511, 536, 497, 509, 521, 542, 490, 495, 546, 510, 530, 502, 545, 495, 516, 498, 494, 502, 500, 513, 489, 521, 498, 533, 524, 508, 506, 486, 521, 500, 529, 508, 497, 527, 555, 517, 477, 470, 500, 493, 506, 513, 521, 546, 501, 482, 538, 529, 487, 535, 507, 524, 515, 522, 475, 546, 516, 478, 528, 504, 521, 496, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 483, 495, 514, 501, 505, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 506, 528, 525, 501, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 503, 507, 564, 520, 523, 498, 23, 4, 0, 0, 0, 0, 0, 0, 485, 504, 472, 533, 501, 493, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 500, 517, 503, 519, 521, 514, 7, 2, 0, 0, 0, 0, 0, 0, 512, 527, 550, 545, 492, 506, 481, 522, 0, 0, 0, 0, 0, 0, 0, 0, 488, 517, 504, 528, 496, 490, 541, 488, 7, 6, 0, 0, 0, 0, 0, 0, 524, 520, 480, 534, 524, 520, 480, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4050, 4178, 4039, 4135, 4135, 4065, 4089] avg=4096.0000 max=4178 max_rank=2 min=4039 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16424 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 534, 510, 504, 487, 498, 494, 504, 0, 0, 0, 0, 0, 0, 0, 0, 500, 535, 537, 502, 526, 509, 523, 477, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 504, 509, 502, 527, 495, 533, 22, 4, 0, 0, 0, 0, 0, 0, 523, 485, 461, 539, 522, 556, 488, 492, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 490, 497, 524, 521, 457, 509, 7, 2, 0, 0, 0, 0, 0, 0, 527, 475, 513, 503, 488, 496, 547, 521, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 530, 507, 490, 545, 538, 520, 8, 7, 0, 0, 0, 0, 0, 0, 488, 503, 558, 505, 531, 494, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4109, 4121, 4066, 4049, 4070, 4183, 4114] avg=4096.0000 max=4183 max_rank=6 min=4049 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16416 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 519, 507, 495, 497, 500, 522, 477, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 496, 501, 506, 493, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 533, 525, 506, 499, 553, 502, 480, 490, 25, 4, 0, 0, 0, 0, 0, 0, 509, 538, 523, 543, 517, 503, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 536, 506, 487, 533, 548, 499, 542, 525, 7, 2, 0, 0, 0, 0, 0, 0, 496, 501, 508, 532, 489, 544, 530, 508, 0, 0, 0, 0, 0, 0, 0, 0, 529, 508, 511, 508, 504, 491, 496, 495, 7, 6, 0, 0, 0, 0, 0, 0, 496, 496, 504, 527, 513, 508, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4031, 4117, 4181, 4185, 4108, 4055, 4063] avg=4096.0000 max=4185 max_rank=4 min=4028 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16411 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 565, 501, 517, 516, 491, 510, 527, 0, 0, 0, 0, 0, 0, 0, 0, 477, 513, 540, 504, 547, 475, 549, 524, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 477, 526, 494, 512, 480, 508, 24, 5, 0, 0, 0, 0, 0, 0, 516, 488, 498, 549, 494, 521, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 485, 490, 527, 505, 538, 476, 526, 501, 7, 2, 0, 0, 0, 0, 0, 0, 500, 525, 489, 537, 516, 509, 496, 493, 0, 0, 0, 0, 0, 0, 0, 0, 517, 472, 539, 509, 528, 538, 516, 536, 8, 6, 0, 0, 0, 0, 0, 0, 510, 510, 519, 505, 472, 491, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4129, 4072, 4076, 4057, 4065, 4169, 4062] avg=4096.0000 max=4169 max_rank=6 min=4057 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16353 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 516, 530, 517, 493, 457, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 475, 488, 521, 561, 523, 495, 515, 542, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 484, 524, 506, 512, 524, 488, 27, 4, 0, 0, 0, 0, 0, 0, 525, 525, 545, 501, 520, 513, 542, 455, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 473, 509, 510, 529, 502, 512, 7, 2, 0, 0, 0, 0, 0, 0, 505, 523, 523, 503, 519, 496, 522, 499, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 530, 516, 517, 533, 524, 535, 7, 6, 0, 0, 0, 0, 0, 0, 505, 486, 500, 485, 511, 556, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4120, 4094, 4126, 4042, 4090, 4156, 4073] avg=4096.0000 max=4156 max_rank=6 min=4042 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16361 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 504, 526, 477, 521, 521, 492, 513, 0, 0, 0, 0, 0, 0, 0, 0, 550, 494, 528, 506, 553, 517, 474, 473, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 537, 496, 493, 515, 493, 481, 25, 4, 0, 0, 0, 0, 0, 0, 491, 491, 523, 488, 540, 531, 521, 514, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 477, 503, 531, 478, 575, 558, 7, 2, 0, 0, 0, 0, 0, 0, 522, 499, 514, 495, 546, 525, 530, 474, 0, 0, 0, 0, 0, 0, 0, 0, 496, 539, 487, 493, 494, 499, 518, 474, 7, 6, 0, 0, 0, 0, 0, 0, 539, 498, 525, 527, 533, 499, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4095, 4073, 4099, 4161, 4105, 4013, 4156] avg=4096.0000 max=4161 max_rank=4 min=4013 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 540, 541, 534, 525, 545, 492, 517, 0, 0, 0, 0, 0, 0, 0, 0, 511, 499, 516, 525, 512, 480, 537, 515, 0, 0, 0, 0, 0, 0, 0, 0, 496, 492, 504, 527, 492, 495, 529, 498, 24, 4, 0, 0, 0, 0, 0, 0, 508, 520, 489, 499, 507, 520, 545, 514, 0, 0, 0, 0, 0, 0, 0, 0, 516, 475, 524, 523, 464, 515, 531, 512, 7, 2, 0, 0, 0, 0, 0, 0, 477, 505, 502, 513, 543, 537, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 512, 473, 501, 471, 522, 509, 8, 6, 0, 0, 0, 0, 0, 0, 459, 493, 532, 532, 511, 513, 573, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 4095, 4061, 4102, 4069, 4113, 4010, 4115] avg=4096.0000 max=4203 max_rank=0 min=4010 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16307 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[548, 490, 536, 497, 509, 521, 542, 490, 0, 0, 0, 0, 0, 0, 0, 0, 495, 546, 510, 530, 502, 545, 495, 516, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 502, 500, 513, 489, 521, 498, 28, 9, 0, 0, 0, 0, 0, 0, 533, 524, 480, 506, 486, 521, 500, 529, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 527, 555, 517, 477, 470, 500, 12, 7, 0, 0, 0, 0, 0, 0, 493, 506, 513, 521, 546, 501, 464, 538, 0, 0, 0, 0, 0, 0, 0, 0, 529, 487, 535, 507, 524, 515, 522, 475, 12, 11, 0, 0, 0, 0, 0, 0, 546, 516, 478, 516, 504, 521, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4139, 4052, 4079, 4070, 4082, 4117, 4096] avg=4096.0000 max=4139 max_rank=1 min=4052 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16365 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 15:43:12] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 273.1 | throughput per GPU (TFLOP/s/GPU): 452.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.119305E+01 | loss scale: 1.0 | grad norm: 107.961 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[510, 527, 526, 501, 504, 493, 505, 550, 542, 519, 505, 475, 512, 506, 520, 539, 555, 509, 514, 531, 496, 508, 512, 510, 540, 514, 513, 466, 491, 505, 509, 503, 515, 505, 520, 454, 486, 547, 553, 506, 524, 489, 523, 499, 498, 498, 548, 510, 527, 547, 520, 510, 529, 493, 483, 503, 506, 498, 504, 488, 539, 525, 492, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[518, 490, 482, 520, 542, 511, 469, 491, 505, 538, 510, 512, 499, 555, 557, 527, 492, 510, 534, 546, 495, 524, 497, 525, 482, 538, 471, 549, 510, 535, 507, 538, 519, 518, 529, 518, 504, 478, 529, 498, 510, 522, 525, 507, 506, 476, 528, 496, 491, 500, 481, 539, 512, 514, 463, 524, 546, 494, 530, 527, 538, 521, 477, 469] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[489, 528, 464, 486, 523, 521, 521, 484, 506, 508, 548, 469, 537, 526, 525, 535, 497, 500, 489, 534, 541, 492, 507, 511, 505, 506, 503, 511, 515, 531, 538, 495, 544, 510, 530, 503, 511, 529, 504, 540, 553, 549, 497, 491, 532, 515, 557, 512, 476, 528, 473, 527, 522, 512, 496, 504, 472, 523, 482, 498, 481, 502, 533, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[509, 503, 521, 494, 512, 507, 518, 503, 502, 499, 517, 494, 528, 519, 505, 505, 497, 544, 509, 483, 499, 504, 524, 530, 529, 534, 521, 509, 487, 499, 548, 510, 526, 493, 528, 511, 516, 479, 543, 504, 545, 559, 526, 463, 497, 523, 519, 465, 508, 500, 547, 531, 537, 532, 494, 479, 508, 554, 469, 516, 491, 473, 544, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[531, 461, 498, 498, 518, 496, 507, 534, 477, 494, 540, 548, 512, 516, 485, 516, 524, 510, 537, 504, 513, 516, 474, 518, 495, 530, 496, 524, 518, 509, 520, 547, 520, 544, 514, 519, 506, 509, 526, 499, 511, 509, 489, 509, 524, 508, 486, 507, 538, 495, 537, 527, 501, 505, 484, 507, 512, 510, 535, 510, 514, 537, 502, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[544, 477, 537, 485, 526, 521, 483, 524, 508, 554, 494, 517, 514, 491, 488, 513, 515, 530, 511, 491, 493, 490, 532, 509, 488, 479, 502, 505, 513, 526, 471, 547, 513, 529, 468, 493, 497, 523, 557, 515, 505, 514, 510, 536, 526, 524, 538, 515, 519, 532, 512, 537, 499, 495, 500, 497, 542, 521, 509, 536, 543, 498, 489, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[523, 526, 517, 493, 551, 525, 505, 523, 509, 508, 508, 505, 536, 521, 482, 536, 525, 513, 525, 499, 543, 520, 548, 472, 471, 527, 496, 501, 524, 512, 506, 546, 513, 530, 492, 438, 538, 523, 526, 545, 523, 523, 466, 525, 499, 499, 539, 493, 504, 512, 478, 510, 499, 511, 457, 530, 510, 499, 498, 515, 504, 541, 527, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[515, 570, 520, 531, 522, 503, 514, 488, 526, 520, 493, 496, 492, 542, 474, 500, 493, 498, 550, 530, 502, 530, 500, 530, 507, 517, 503, 483, 515, 512, 520, 498, 500, 528, 514, 499, 541, 508, 489, 494, 531, 487, 525, 518, 520, 529, 521, 499, 560, 491, 509, 513, 521, 496, 547, 490, 516, 478, 496, 511, 519, 500, 522, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 527, 526, 501, 504, 493, 505, 550, 5, 4, 0, 0, 0, 0, 0, 0, 542, 519, 505, 475, 512, 496, 520, 539, 0, 0, 0, 0, 0, 0, 0, 0, 555, 509, 503, 531, 496, 508, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 540, 514, 513, 466, 491, 505, 509, 503, 5, 0, 0, 0, 0, 0, 0, 0, 515, 505, 520, 454, 486, 547, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 489, 523, 499, 498, 498, 535, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 547, 520, 510, 529, 493, 483, 503, 13, 6, 0, 0, 0, 0, 0, 0, 506, 498, 504, 488, 539, 525, 492, 519, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4108, 4124, 4046, 4075, 4076, 4131, 4083] avg=4096.0000 max=4131 max_rank=6 min=4046 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 503, 521, 494, 512, 507, 518, 503, 5, 4, 0, 0, 0, 0, 0, 0, 502, 499, 517, 494, 528, 509, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 497, 544, 498, 483, 499, 504, 524, 530, 0, 0, 0, 0, 0, 0, 0, 0, 529, 534, 521, 509, 487, 499, 548, 510, 5, 0, 0, 0, 0, 0, 0, 0, 526, 493, 528, 511, 516, 479, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 545, 559, 526, 463, 497, 523, 507, 465, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 547, 531, 537, 532, 494, 479, 12, 6, 0, 0, 0, 0, 0, 0, 508, 554, 469, 516, 491, 473, 544, 525, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4059, 4079, 4142, 4089, 4085, 4146, 4092] avg=4096.0000 max=4146 max_rank=6 min=4059 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 482, 520, 542, 511, 469, 491, 5, 4, 0, 0, 0, 0, 0, 0, 505, 538, 510, 512, 499, 544, 557, 527, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 523, 546, 495, 524, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 538, 471, 549, 510, 535, 507, 538, 6, 0, 0, 0, 0, 0, 0, 0, 519, 518, 529, 518, 504, 478, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 525, 507, 506, 476, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 491, 500, 481, 539, 512, 514, 463, 524, 12, 6, 0, 0, 0, 0, 0, 0, 546, 494, 530, 527, 538, 521, 477, 469, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4192, 4112, 4136, 4083, 4058, 4042, 4113] avg=4096.0000 max=4192 max_rank=1 min=4032 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[489, 528, 464, 486, 523, 521, 521, 484, 5, 4, 0, 0, 0, 0, 0, 0, 506, 508, 548, 469, 537, 515, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 497, 500, 479, 534, 541, 492, 507, 511, 0, 0, 0, 0, 0, 0, 0, 0, 505, 506, 503, 511, 515, 531, 538, 495, 6, 0, 0, 0, 0, 0, 0, 0, 544, 510, 530, 503, 511, 529, 494, 540, 0, 0, 0, 0, 0, 0, 0, 0, 553, 549, 497, 491, 532, 515, 544, 512, 0, 0, 0, 0, 0, 0, 0, 0, 476, 528, 473, 527, 522, 512, 496, 504, 13, 5, 0, 0, 0, 0, 0, 0, 472, 523, 482, 498, 481, 502, 533, 517, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4143, 4061, 4110, 4161, 4193, 4056, 4019] avg=4096.0000 max=4193 max_rank=5 min=4019 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[531, 461, 498, 498, 518, 496, 507, 534, 5, 4, 0, 0, 0, 0, 0, 0, 477, 494, 540, 548, 512, 506, 485, 516, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 526, 504, 513, 516, 474, 518, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 496, 524, 518, 509, 520, 547, 5, 0, 0, 0, 0, 0, 0, 0, 520, 544, 514, 519, 506, 509, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 489, 509, 524, 508, 475, 507, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 537, 527, 501, 505, 484, 507, 11, 6, 0, 0, 0, 0, 0, 0, 512, 510, 535, 510, 514, 537, 502, 508, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4078, 4085, 4144, 4127, 4032, 4111, 4139] avg=4096.0000 max=4144 max_rank=3 min=4032 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 526, 517, 493, 551, 525, 505, 523, 5, 4, 0, 0, 0, 0, 0, 0, 509, 508, 508, 505, 536, 511, 482, 536, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 514, 499, 543, 520, 548, 472, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 496, 501, 524, 512, 506, 546, 5, 0, 0, 0, 0, 0, 0, 0, 513, 530, 492, 438, 538, 523, 516, 545, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 466, 525, 499, 499, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 504, 512, 478, 510, 499, 511, 457, 530, 12, 6, 0, 0, 0, 0, 0, 0, 510, 499, 498, 515, 504, 541, 527, 505, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4095, 4134, 4088, 4095, 4055, 4019, 4110] avg=4096.0000 max=4172 max_rank=0 min=4019 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 570, 520, 531, 522, 503, 514, 488, 5, 4, 0, 0, 0, 0, 0, 0, 526, 520, 493, 496, 492, 531, 474, 500, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 539, 530, 502, 530, 500, 530, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 503, 483, 515, 512, 520, 498, 6, 0, 0, 0, 0, 0, 0, 0, 500, 528, 514, 499, 541, 508, 479, 494, 0, 0, 0, 0, 0, 0, 0, 0, 531, 487, 525, 518, 520, 529, 509, 499, 0, 0, 0, 0, 0, 0, 0, 0, 560, 491, 509, 513, 521, 496, 547, 490, 12, 6, 0, 0, 0, 0, 0, 0, 516, 478, 496, 511, 519, 500, 522, 502, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4032, 4122, 4061, 4063, 4118, 4145, 4055] avg=4096.0000 max=4172 max_rank=0 min=4032 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[544, 477, 537, 485, 526, 521, 483, 524, 8, 9, 0, 0, 0, 0, 0, 0, 508, 554, 494, 517, 514, 473, 488, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 491, 491, 493, 490, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 488, 479, 502, 505, 513, 526, 471, 547, 10, 0, 0, 0, 0, 0, 0, 0, 513, 529, 468, 493, 497, 523, 537, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 510, 536, 526, 524, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 512, 537, 499, 495, 500, 497, 17, 10, 0, 0, 0, 0, 0, 0, 542, 521, 509, 536, 543, 498, 489, 498, 11, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4061, 4051, 4041, 4075, 4151, 4118, 4157] avg=4096.0000 max=4157 max_rank=7 min=4041 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[497, 534, 526, 536, 513, 521, 474, 510, 494, 493, 525, 510, 525, 519, 531, 555, 485, 514, 515, 540, 502, 515, 524, 489, 540, 510, 495, 534, 525, 514, 516, 476, 491, 487, 487, 521, 531, 522, 532, 534, 503, 567, 467, 483, 485, 499, 479, 494, 543, 505, 493, 510, 523, 545, 516, 538, 506, 546, 506, 519, 489, 498, 512, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[486, 499, 534, 496, 505, 516, 529, 512, 531, 497, 511, 519, 535, 557, 484, 533, 530, 514, 499, 540, 504, 550, 509, 503, 514, 502, 553, 518, 496, 530, 509, 516, 475, 519, 531, 500, 548, 531, 511, 505, 507, 489, 509, 499, 489, 477, 530, 497, 536, 508, 503, 509, 498, 496, 530, 500, 496, 522, 504, 512, 493, 532, 508, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[509, 512, 508, 502, 530, 542, 527, 504, 503, 523, 484, 488, 512, 497, 512, 488, 495, 490, 508, 509, 564, 508, 484, 487, 534, 523, 520, 505, 517, 526, 532, 462, 511, 521, 497, 523, 479, 520, 520, 515, 477, 517, 549, 520, 528, 488, 533, 529, 527, 477, 512, 519, 527, 519, 508, 502, 516, 537, 517, 530, 489, 510, 511, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[488, 553, 507, 476, 510, 488, 504, 487, 553, 528, 520, 530, 495, 480, 492, 515, 493, 499, 526, 469, 507, 537, 515, 507, 522, 528, 535, 531, 517, 507, 529, 524, 545, 519, 542, 482, 502, 504, 503, 505, 509, 546, 472, 516, 477, 492, 520, 515, 491, 533, 520, 540, 509, 491, 501, 491, 527, 550, 539, 517, 495, 495, 535, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[502, 516, 530, 518, 488, 505, 494, 502, 508, 519, 492, 544, 519, 495, 518, 503, 504, 499, 514, 470, 518, 521, 547, 530, 531, 525, 514, 509, 492, 512, 548, 497, 508, 562, 529, 494, 531, 510, 504, 485, 525, 507, 502, 533, 551, 508, 534, 503, 532, 489, 499, 477, 475, 517, 520, 479, 505, 506, 470, 527, 552, 524, 515, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[533, 497, 527, 515, 539, 484, 506, 519, 505, 562, 471, 486, 543, 490, 465, 497, 529, 485, 494, 497, 530, 546, 518, 460, 521, 498, 548, 527, 476, 560, 505, 546, 501, 526, 517, 512, 505, 500, 515, 514, 510, 504, 512, 520, 516, 503, 523, 522, 512, 513, 534, 494, 498, 503, 497, 519, 492, 502, 519, 486, 511, 548, 513, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[517, 541, 507, 528, 547, 505, 504, 520, 478, 500, 550, 544, 498, 523, 501, 491, 520, 477, 471, 474, 496, 474, 518, 537, 513, 512, 524, 485, 530, 513, 480, 503, 510, 459, 450, 522, 499, 514, 523, 516, 514, 558, 518, 501, 522, 483, 549, 498, 542, 504, 504, 546, 480, 542, 506, 499, 548, 553, 498, 537, 553, 495, 511, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[484, 514, 505, 527, 498, 509, 486, 515, 560, 518, 532, 531, 515, 509, 499, 491, 506, 508, 544, 539, 512, 508, 502, 490, 500, 493, 530, 530, 523, 478, 470, 544, 530, 499, 475, 510, 501, 508, 528, 509, 509, 483, 535, 542, 541, 524, 498, 460, 501, 532, 511, 506, 536, 555, 510, 492, 553, 466, 520, 527, 509, 506, 514, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 534, 526, 536, 513, 521, 474, 510, 6, 0, 0, 0, 0, 0, 0, 0, 494, 493, 525, 508, 525, 519, 531, 555, 0, 0, 0, 0, 0, 0, 0, 0, 485, 514, 515, 540, 502, 515, 524, 489, 23, 0, 0, 0, 0, 0, 0, 0, 540, 510, 469, 534, 525, 514, 516, 476, 0, 0, 0, 0, 0, 0, 0, 0, 491, 487, 487, 521, 531, 522, 532, 534, 11, 0, 0, 0, 0, 0, 0, 0, 503, 567, 467, 483, 485, 499, 479, 494, 8, 3, 0, 0, 0, 0, 0, 0, 543, 505, 493, 510, 523, 545, 516, 538, 2, 0, 0, 0, 0, 0, 0, 0, 506, 521, 506, 519, 489, 498, 512, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4150, 4107, 4084, 4116, 3988, 4175, 4031] avg=4096.0000 max=4175 max_rank=6 min=3988 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[488, 553, 507, 476, 510, 488, 504, 487, 6, 0, 0, 0, 0, 0, 0, 0, 553, 528, 520, 527, 495, 480, 492, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 499, 526, 469, 507, 537, 515, 507, 25, 0, 0, 0, 0, 0, 0, 0, 522, 528, 507, 531, 517, 507, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 542, 482, 502, 504, 503, 505, 11, 0, 0, 0, 0, 0, 0, 0, 509, 546, 472, 516, 477, 492, 520, 515, 8, 3, 0, 0, 0, 0, 0, 0, 491, 533, 520, 540, 509, 491, 501, 491, 3, 0, 0, 0, 0, 0, 0, 0, 527, 525, 539, 517, 495, 495, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4110, 4078, 4165, 4113, 4058, 4079, 4146] avg=4096.0000 max=4165 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 512, 508, 502, 530, 542, 527, 504, 6, 0, 0, 0, 0, 0, 0, 0, 503, 523, 484, 486, 512, 497, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 508, 509, 564, 508, 484, 487, 24, 0, 0, 0, 0, 0, 0, 0, 534, 523, 493, 505, 517, 526, 532, 462, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 497, 523, 479, 520, 520, 515, 11, 0, 0, 0, 0, 0, 0, 0, 477, 517, 549, 520, 528, 488, 533, 529, 8, 3, 0, 0, 0, 0, 0, 0, 527, 477, 512, 519, 527, 519, 508, 502, 2, 0, 0, 0, 0, 0, 0, 0, 516, 512, 517, 530, 489, 510, 511, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4005, 4069, 4092, 4097, 4152, 4093, 4120] avg=4096.0000 max=4152 max_rank=5 min=4005 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[486, 499, 534, 496, 505, 516, 529, 512, 5, 0, 0, 0, 0, 0, 0, 0, 531, 497, 511, 516, 535, 557, 484, 533, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 499, 540, 504, 550, 509, 503, 26, 0, 0, 0, 0, 0, 0, 0, 514, 502, 524, 518, 496, 530, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 475, 519, 531, 500, 548, 531, 511, 505, 11, 0, 0, 0, 0, 0, 0, 0, 507, 489, 509, 499, 489, 477, 530, 497, 8, 3, 0, 0, 0, 0, 0, 0, 536, 508, 503, 509, 498, 496, 530, 500, 3, 0, 0, 0, 0, 0, 0, 0, 496, 498, 504, 512, 493, 532, 508, 473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4164, 4175, 4109, 4131, 4008, 4083, 4016] avg=4096.0000 max=4175 max_rank=2 min=4008 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 516, 530, 518, 488, 505, 494, 502, 5, 0, 0, 0, 0, 0, 0, 0, 508, 519, 492, 541, 519, 495, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 504, 499, 514, 470, 518, 521, 547, 530, 24, 0, 0, 0, 0, 0, 0, 0, 531, 525, 487, 509, 492, 512, 548, 497, 0, 0, 0, 0, 0, 0, 0, 0, 508, 562, 529, 494, 531, 510, 504, 485, 10, 0, 0, 0, 0, 0, 0, 0, 525, 507, 502, 533, 551, 508, 534, 503, 8, 3, 0, 0, 0, 0, 0, 0, 532, 489, 499, 477, 475, 517, 520, 479, 3, 0, 0, 0, 0, 0, 0, 0, 505, 483, 470, 527, 552, 524, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4095, 4127, 4101, 4133, 4174, 3991, 4087] avg=4096.0000 max=4174 max_rank=5 min=3991 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 541, 507, 528, 547, 505, 504, 520, 6, 0, 0, 0, 0, 0, 0, 0, 478, 500, 550, 541, 498, 523, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 520, 477, 471, 474, 496, 474, 518, 537, 24, 0, 0, 0, 0, 0, 0, 0, 513, 512, 497, 485, 530, 513, 480, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 459, 450, 522, 499, 514, 523, 516, 11, 0, 0, 0, 0, 0, 0, 0, 514, 558, 518, 501, 522, 483, 549, 498, 8, 3, 0, 0, 0, 0, 0, 0, 542, 504, 504, 546, 480, 542, 506, 499, 3, 0, 0, 0, 0, 0, 0, 0, 548, 528, 498, 537, 553, 495, 511, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4082, 3991, 4033, 4004, 4154, 4126, 4203] avg=4096.0000 max=4203 max_rank=7 min=3991 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[533, 497, 527, 515, 539, 484, 506, 519, 5, 0, 0, 0, 0, 0, 0, 0, 505, 562, 471, 484, 543, 490, 465, 497, 0, 0, 0, 0, 0, 0, 0, 0, 529, 485, 494, 497, 530, 546, 518, 460, 25, 0, 0, 0, 0, 0, 0, 0, 521, 498, 520, 527, 476, 560, 505, 546, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 517, 512, 505, 500, 515, 514, 10, 0, 0, 0, 0, 0, 0, 0, 510, 504, 512, 520, 516, 503, 523, 522, 8, 3, 0, 0, 0, 0, 0, 0, 512, 513, 534, 494, 498, 503, 497, 519, 2, 0, 0, 0, 0, 0, 0, 0, 492, 479, 519, 486, 511, 548, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4017, 4084, 4153, 4100, 4121, 4072, 4096] avg=4096.0000 max=4153 max_rank=3 min=4017 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[484, 514, 505, 527, 498, 509, 486, 515, 8, 4, 0, 0, 0, 0, 0, 0, 560, 518, 532, 521, 515, 509, 499, 491, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 544, 539, 512, 508, 502, 490, 28, 0, 0, 0, 0, 0, 0, 0, 500, 493, 493, 530, 523, 478, 470, 544, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 475, 510, 501, 508, 528, 509, 14, 0, 0, 0, 0, 0, 0, 0, 509, 483, 535, 542, 541, 524, 498, 460, 12, 9, 0, 0, 0, 0, 0, 0, 501, 532, 511, 506, 536, 555, 510, 492, 6, 0, 0, 0, 0, 0, 0, 0, 553, 432, 520, 527, 509, 506, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4145, 4137, 4031, 4074, 4113, 4149, 4069] avg=4096.0000 max=4149 max_rank=6 min=4031 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[486, 521, 504, 498, 535, 471, 483, 523, 469, 553, 485, 525, 568, 521, 499, 523, 548, 489, 493, 518, 509, 485, 484, 475, 474, 548, 509, 510, 549, 539, 507, 523, 536, 516, 507, 528, 523, 535, 537, 504, 500, 516, 494, 496, 498, 524, 525, 512, 500, 488, 526, 546, 546, 509, 536, 517, 511, 496, 506, 496, 536, 489, 505, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[533, 553, 504, 513, 491, 500, 504, 521, 532, 541, 497, 544, 534, 504, 511, 498, 493, 508, 531, 524, 503, 548, 520, 495, 509, 479, 569, 493, 504, 539, 492, 487, 510, 493, 519, 534, 548, 486, 539, 557, 488, 506, 495, 553, 467, 505, 490, 518, 495, 545, 502, 504, 532, 478, 537, 451, 533, 490, 519, 468, 484, 536, 498, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[515, 523, 555, 548, 545, 535, 490, 491, 508, 462, 499, 524, 528, 509, 558, 529, 506, 559, 508, 498, 487, 522, 477, 521, 506, 482, 520, 541, 495, 501, 526, 508, 501, 521, 497, 496, 545, 496, 500, 513, 572, 514, 544, 529, 501, 498, 503, 512, 486, 542, 513, 512, 531, 489, 523, 471, 505, 502, 507, 524, 472, 479, 504, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[495, 521, 527, 504, 518, 483, 498, 537, 479, 539, 569, 520, 521, 547, 511, 496, 491, 485, 534, 534, 521, 523, 508, 535, 533, 511, 501, 495, 491, 509, 529, 510, 473, 511, 531, 498, 510, 495, 540, 509, 509, 486, 514, 481, 532, 512, 481, 531, 505, 522, 522, 522, 505, 491, 526, 507, 512, 529, 517, 480, 506, 521, 489, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[472, 505, 518, 549, 542, 512, 502, 509, 495, 552, 540, 526, 510, 510, 535, 521, 477, 480, 506, 504, 515, 509, 557, 523, 493, 507, 486, 521, 513, 502, 527, 508, 513, 458, 509, 527, 494, 558, 504, 510, 509, 531, 531, 523, 472, 522, 532, 519, 486, 512, 534, 525, 500, 510, 499, 507, 535, 503, 490, 528, 521, 495, 490, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[526, 495, 539, 541, 510, 513, 537, 478, 475, 530, 465, 510, 511, 542, 525, 514, 500, 495, 503, 515, 493, 519, 504, 517, 540, 524, 466, 511, 524, 505, 533, 506, 536, 526, 498, 519, 510, 519, 542, 509, 509, 497, 476, 520, 512, 546, 495, 480, 475, 484, 552, 500, 489, 511, 513, 517, 489, 531, 487, 544, 499, 575, 523, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[492, 495, 524, 435, 506, 507, 482, 517, 515, 505, 494, 486, 514, 506, 539, 533, 513, 522, 515, 523, 504, 501, 485, 533, 526, 541, 523, 529, 529, 487, 530, 450, 541, 505, 525, 509, 532, 480, 524, 502, 493, 493, 539, 521, 481, 520, 516, 502, 528, 539, 537, 512, 535, 497, 502, 550, 537, 486, 497, 508, 523, 523, 517, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[535, 490, 511, 490, 535, 523, 489, 471, 505, 552, 505, 517, 517, 480, 517, 511, 480, 523, 507, 525, 493, 534, 516, 493, 513, 483, 533, 539, 551, 533, 538, 462, 520, 526, 495, 467, 536, 519, 534, 531, 532, 546, 509, 540, 506, 483, 488, 526, 492, 516, 508, 483, 560, 528, 520, 465, 520, 514, 504, 508, 519, 525, 489, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 521, 527, 504, 518, 483, 498, 537, 11, 0, 0, 0, 0, 0, 0, 0, 479, 503, 569, 520, 521, 547, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 534, 534, 521, 523, 508, 535, 10, 8, 0, 0, 0, 0, 0, 0, 533, 511, 501, 495, 491, 509, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0, 473, 511, 531, 498, 510, 495, 514, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 514, 481, 532, 512, 481, 531, 6, 5, 0, 0, 0, 0, 0, 0, 505, 522, 522, 522, 505, 491, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 512, 529, 517, 480, 506, 521, 489, 526, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4146, 4149, 4073, 4041, 4057, 4100, 4108] avg=4096.0000 max=4149 max_rank=2 min=4041 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[486, 521, 504, 498, 535, 471, 483, 523, 11, 0, 0, 0, 0, 0, 0, 0, 469, 515, 485, 525, 568, 521, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 548, 489, 493, 518, 509, 485, 484, 475, 10, 9, 0, 0, 0, 0, 0, 0, 474, 548, 509, 510, 549, 539, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 507, 528, 523, 535, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 494, 496, 498, 524, 525, 512, 6, 5, 0, 0, 0, 0, 0, 0, 500, 488, 526, 546, 546, 509, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 506, 496, 536, 489, 505, 486, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4105, 4020, 4153, 4160, 4076, 4168, 4054] avg=4096.0000 max=4168 max_rank=6 min=4020 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[526, 495, 539, 541, 510, 513, 537, 478, 11, 0, 0, 0, 0, 0, 0, 0, 475, 495, 465, 510, 511, 542, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 503, 515, 493, 519, 504, 517, 10, 8, 0, 0, 0, 0, 0, 0, 540, 524, 466, 511, 524, 505, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 536, 526, 498, 519, 510, 519, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 476, 520, 512, 546, 495, 480, 6, 5, 0, 0, 0, 0, 0, 0, 475, 484, 552, 500, 489, 511, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 489, 531, 487, 544, 499, 575, 523, 519, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4037, 4064, 4103, 4133, 4046, 4041, 4194] avg=4096.0000 max=4194 max_rank=7 min=4037 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[535, 490, 511, 490, 535, 523, 489, 471, 11, 0, 0, 0, 0, 0, 0, 0, 505, 515, 505, 517, 517, 480, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 507, 525, 493, 534, 516, 493, 10, 8, 0, 0, 0, 0, 0, 0, 513, 483, 533, 539, 551, 533, 532, 462, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 495, 467, 536, 519, 508, 531, 0, 0, 0, 0, 0, 0, 0, 0, 532, 546, 509, 540, 506, 483, 488, 526, 6, 5, 0, 0, 0, 0, 0, 0, 492, 516, 508, 483, 560, 528, 520, 465, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 504, 508, 519, 525, 489, 488, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4067, 4089, 4146, 4102, 4141, 4072, 4096] avg=4096.0000 max=4146 max_rank=3 min=4055 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[533, 553, 504, 513, 491, 500, 504, 521, 11, 0, 0, 0, 0, 0, 0, 0, 532, 505, 497, 544, 534, 504, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 493, 508, 531, 524, 503, 548, 520, 495, 10, 8, 0, 0, 0, 0, 0, 0, 509, 479, 569, 493, 504, 539, 486, 487, 0, 0, 0, 0, 0, 0, 0, 0, 510, 493, 519, 534, 548, 486, 513, 557, 0, 0, 0, 0, 0, 0, 0, 0, 488, 506, 495, 553, 467, 505, 490, 518, 6, 5, 0, 0, 0, 0, 0, 0, 495, 545, 502, 504, 532, 478, 537, 451, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 519, 468, 484, 536, 498, 514, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4125, 4140, 4066, 4160, 4033, 4044, 4070] avg=4096.0000 max=4160 max_rank=4 min=4033 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 523, 555, 548, 545, 535, 490, 491, 16, 0, 0, 0, 0, 0, 0, 0, 508, 423, 499, 524, 528, 509, 558, 529, 0, 0, 0, 0, 0, 0, 0, 0, 506, 559, 508, 498, 487, 522, 477, 521, 14, 12, 0, 0, 0, 0, 0, 0, 506, 482, 520, 541, 495, 501, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 497, 496, 545, 496, 463, 513, 0, 0, 0, 0, 0, 0, 0, 0, 572, 514, 544, 529, 501, 498, 503, 512, 9, 7, 0, 0, 0, 0, 0, 0, 486, 542, 513, 512, 531, 489, 523, 471, 3, 0, 0, 0, 0, 0, 0, 0, 505, 502, 507, 524, 472, 479, 504, 490, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4218, 4078, 4104, 4067, 4032, 4189, 4070, 4010] avg=4096.0000 max=4218 max_rank=0 min=4010 min_rank=7 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[472, 505, 518, 549, 542, 512, 502, 509, 11, 0, 0, 0, 0, 0, 0, 0, 495, 515, 540, 526, 510, 510, 535, 521, 0, 0, 0, 0, 0, 0, 0, 0, 477, 480, 506, 504, 515, 509, 557, 523, 9, 8, 0, 0, 0, 0, 0, 0, 493, 507, 486, 521, 513, 502, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 513, 458, 509, 527, 494, 558, 479, 510, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 531, 523, 472, 522, 532, 519, 6, 5, 0, 0, 0, 0, 0, 0, 486, 512, 534, 525, 500, 510, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 490, 528, 521, 495, 490, 495, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4152, 4088, 4051, 4048, 4150, 4073, 4086] avg=4096.0000 max=4152 max_rank=1 min=4048 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 495, 524, 435, 506, 507, 482, 517, 11, 0, 0, 0, 0, 0, 0, 0, 515, 471, 494, 486, 514, 506, 539, 533, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 515, 523, 504, 501, 485, 533, 10, 8, 0, 0, 0, 0, 0, 0, 526, 541, 523, 529, 529, 487, 524, 450, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 525, 509, 532, 480, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 539, 521, 481, 520, 516, 502, 6, 5, 0, 0, 0, 0, 0, 0, 528, 539, 537, 512, 535, 497, 502, 550, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 497, 508, 523, 523, 517, 533, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4058, 4114, 4109, 4092, 4076, 4200, 4150] avg=4096.0000 max=4200 max_rank=6 min=3969 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[471, 518, 519, 510, 512, 548, 512, 482, 529, 502, 512, 525, 518, 471, 515, 491, 517, 505, 528, 521, 539, 499, 527, 563, 553, 499, 519, 509, 527, 484, 509, 468, 487, 503, 509, 487, 479, 516, 503, 541, 519, 517, 533, 515, 488, 537, 485, 504, 496, 517, 513, 513, 500, 521, 493, 513, 506, 520, 489, 526, 535, 533, 505, 563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[505, 493, 558, 500, 528, 521, 493, 518, 525, 483, 522, 545, 520, 515, 503, 526, 531, 499, 568, 502, 518, 508, 551, 541, 484, 507, 498, 497, 535, 479, 531, 513, 480, 533, 503, 497, 511, 530, 504, 506, 485, 485, 538, 514, 520, 534, 493, 484, 504, 546, 532, 497, 516, 531, 512, 499, 523, 488, 499, 469, 524, 492, 474, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[496, 505, 521, 525, 518, 505, 482, 552, 509, 539, 509, 543, 515, 508, 512, 523, 541, 527, 543, 516, 495, 496, 469, 480, 512, 485, 540, 513, 512, 501, 516, 487, 544, 527, 478, 508, 525, 528, 547, 519, 513, 519, 454, 530, 517, 501, 475, 505, 504, 501, 509, 510, 513, 512, 511, 478, 501, 525, 531, 497, 471, 550, 540, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[550, 526, 502, 479, 508, 492, 517, 505, 491, 500, 498, 485, 529, 510, 498, 480, 512, 479, 540, 509, 495, 526, 522, 552, 511, 531, 541, 490, 504, 518, 527, 507, 515, 529, 513, 523, 517, 522, 527, 544, 525, 524, 504, 488, 534, 543, 518, 506, 498, 509, 527, 491, 492, 523, 511, 492, 546, 510, 481, 487, 547, 525, 483, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[476, 536, 512, 492, 564, 532, 545, 488, 531, 527, 501, 497, 515, 516, 554, 481, 538, 534, 566, 493, 521, 508, 523, 535, 525, 483, 525, 520, 517, 500, 445, 469, 483, 498, 472, 501, 525, 537, 494, 516, 509, 527, 472, 497, 495, 519, 512, 497, 500, 533, 518, 541, 482, 497, 493, 486, 520, 522, 537, 547, 534, 493, 501, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[531, 500, 542, 530, 547, 522, 513, 488, 506, 532, 523, 477, 549, 543, 497, 497, 519, 545, 532, 536, 494, 464, 525, 469, 529, 487, 488, 533, 510, 522, 525, 538, 462, 522, 520, 481, 519, 539, 506, 495, 510, 507, 505, 502, 528, 546, 528, 488, 471, 528, 537, 478, 529, 491, 508, 491, 493, 475, 541, 514, 541, 531, 497, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[505, 536, 514, 523, 515, 514, 470, 526, 569, 527, 483, 512, 525, 532, 521, 519, 508, 509, 495, 523, 517, 488, 521, 560, 517, 507, 526, 502, 475, 490, 483, 538, 563, 494, 504, 522, 490, 525, 487, 495, 545, 487, 529, 480, 504, 504, 520, 507, 489, 482, 517, 515, 483, 536, 491, 507, 507, 515, 555, 537, 507, 523, 506, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[544, 525, 486, 539, 506, 549, 496, 527, 503, 533, 519, 527, 515, 505, 511, 515, 537, 514, 493, 517, 522, 500, 556, 475, 486, 492, 526, 477, 509, 492, 521, 545, 521, 527, 523, 520, 495, 533, 516, 505, 523, 496, 470, 516, 519, 517, 503, 491, 493, 497, 516, 487, 476, 558, 528, 506, 501, 481, 519, 529, 491, 503, 526, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[496, 505, 521, 525, 495, 505, 482, 552, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 509, 543, 498, 508, 512, 523, 0, 0, 0, 0, 0, 0, 0, 0, 541, 527, 495, 516, 495, 496, 469, 480, 0, 0, 0, 0, 0, 0, 0, 0, 512, 485, 540, 513, 512, 501, 516, 487, 23, 8, 0, 0, 0, 0, 0, 0, 544, 527, 478, 508, 525, 528, 547, 519, 2, 0, 0, 0, 0, 0, 0, 0, 513, 519, 454, 530, 517, 501, 475, 505, 17, 7, 0, 0, 0, 0, 0, 0, 504, 501, 509, 510, 513, 512, 511, 478, 40, 0, 0, 0, 0, 0, 0, 0, 501, 525, 522, 497, 471, 550, 540, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4141, 4019, 4097, 4178, 4038, 4078, 4136] avg=4096.0000 max=4178 max_rank=4 min=4019 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[476, 536, 512, 492, 539, 532, 545, 488, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 501, 497, 498, 516, 554, 481, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 515, 493, 521, 508, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 525, 520, 517, 500, 445, 469, 25, 9, 0, 0, 0, 0, 0, 0, 483, 498, 472, 501, 525, 537, 494, 516, 2, 0, 0, 0, 0, 0, 0, 0, 509, 527, 472, 497, 495, 519, 512, 497, 17, 7, 0, 0, 0, 0, 0, 0, 500, 533, 518, 541, 482, 497, 493, 486, 42, 0, 0, 0, 0, 0, 0, 0, 520, 522, 528, 547, 534, 493, 501, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4105, 4167, 4018, 4028, 4052, 4092, 4186] avg=4096.0000 max=4186 max_rank=7 min=4018 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[471, 518, 519, 510, 489, 548, 512, 482, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 512, 525, 501, 471, 515, 491, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 481, 521, 539, 499, 527, 563, 0, 0, 0, 0, 0, 0, 0, 0, 553, 499, 519, 509, 527, 484, 509, 468, 23, 8, 0, 0, 0, 0, 0, 0, 487, 503, 509, 487, 479, 516, 503, 541, 2, 0, 0, 0, 0, 0, 0, 0, 519, 517, 533, 515, 488, 537, 485, 504, 17, 6, 0, 0, 0, 0, 0, 0, 496, 517, 513, 513, 500, 521, 493, 513, 39, 0, 0, 0, 0, 0, 0, 0, 506, 520, 481, 526, 535, 533, 505, 563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4046, 4152, 4099, 4027, 4121, 4105, 4169] avg=4096.0000 max=4169 max_rank=7 min=4027 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[531, 500, 542, 530, 523, 522, 513, 488, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 523, 477, 530, 543, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 519, 545, 485, 536, 494, 464, 525, 469, 0, 0, 0, 0, 0, 0, 0, 0, 529, 487, 488, 533, 510, 522, 525, 538, 24, 8, 0, 0, 0, 0, 0, 0, 462, 522, 520, 481, 519, 539, 506, 495, 2, 0, 0, 0, 0, 0, 0, 0, 510, 507, 505, 502, 528, 546, 528, 488, 19, 7, 0, 0, 0, 0, 0, 0, 471, 528, 537, 478, 529, 491, 508, 491, 39, 0, 0, 0, 0, 0, 0, 0, 493, 475, 532, 514, 541, 531, 497, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4105, 4037, 4164, 4046, 4140, 4072, 4055] avg=4096.0000 max=4164 max_rank=3 min=4037 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[505, 493, 558, 500, 504, 521, 493, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 522, 545, 502, 515, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 531, 499, 517, 502, 518, 508, 551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 484, 507, 498, 497, 535, 479, 531, 513, 24, 9, 0, 0, 0, 0, 0, 0, 480, 533, 503, 497, 511, 530, 504, 506, 2, 0, 0, 0, 0, 0, 0, 0, 485, 485, 538, 514, 520, 534, 493, 484, 18, 6, 0, 0, 0, 0, 0, 0, 504, 546, 532, 497, 516, 531, 512, 499, 42, 0, 0, 0, 0, 0, 0, 0, 523, 488, 491, 469, 524, 492, 474, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4121, 4167, 4077, 4066, 4077, 4179, 3989] avg=4096.0000 max=4179 max_rank=6 min=3989 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 526, 502, 479, 485, 492, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 500, 498, 485, 511, 510, 498, 480, 0, 0, 0, 0, 0, 0, 0, 0, 512, 479, 492, 509, 495, 526, 522, 552, 0, 0, 0, 0, 0, 0, 0, 0, 511, 531, 541, 490, 504, 518, 527, 507, 23, 8, 0, 0, 0, 0, 0, 0, 515, 529, 513, 523, 517, 522, 527, 544, 2, 0, 0, 0, 0, 0, 0, 0, 525, 524, 504, 488, 534, 543, 518, 506, 18, 6, 0, 0, 0, 0, 0, 0, 498, 509, 527, 491, 492, 523, 511, 492, 40, 0, 0, 0, 0, 0, 0, 0, 546, 510, 473, 487, 547, 525, 483, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 3973, 4087, 4160, 4192, 4166, 4083, 4051] avg=4096.0000 max=4192 max_rank=4 min=3973 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[505, 536, 514, 523, 484, 514, 470, 526, 0, 0, 0, 0, 0, 0, 0, 0, 569, 527, 483, 512, 503, 532, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 442, 523, 517, 488, 521, 560, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 526, 502, 475, 490, 483, 538, 26, 11, 0, 0, 0, 0, 0, 0, 563, 494, 504, 522, 490, 525, 487, 495, 9, 0, 0, 0, 0, 0, 0, 0, 545, 487, 529, 480, 504, 504, 520, 507, 22, 12, 5, 0, 0, 0, 0, 0, 489, 482, 517, 515, 483, 536, 491, 507, 42, 0, 0, 0, 0, 0, 0, 0, 507, 515, 534, 537, 507, 523, 506, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4166, 4068, 4075, 4089, 4115, 4062, 4121] avg=4096.0000 max=4166 max_rank=1 min=4062 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[544, 525, 486, 539, 483, 549, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 503, 533, 519, 527, 498, 505, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 537, 514, 450, 517, 522, 500, 556, 475, 0, 0, 0, 0, 0, 0, 0, 0, 486, 492, 526, 477, 509, 492, 521, 545, 23, 7, 0, 0, 0, 0, 0, 0, 521, 527, 523, 520, 495, 533, 516, 505, 2, 0, 0, 0, 0, 0, 0, 0, 523, 496, 470, 516, 519, 517, 503, 491, 17, 7, 0, 0, 0, 0, 0, 0, 493, 497, 516, 487, 476, 558, 528, 506, 36, 0, 0, 0, 0, 0, 0, 0, 501, 481, 510, 529, 491, 503, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4111, 4071, 4078, 4142, 4059, 4097, 4061] avg=4096.0000 max=4149 max_rank=0 min=4059 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4352, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[511, 500, 533, 535, 517, 530, 485, 521, 489, 504, 488, 534, 514, 482, 468, 512, 511, 517, 531, 523, 521, 541, 527, 507, 461, 505, 516, 547, 555, 508, 537, 499, 529, 532, 543, 546, 539, 508, 517, 512, 485, 511, 507, 472, 559, 459, 523, 483, 522, 485, 523, 521, 503, 495, 508, 509, 537, 474, 496, 492, 517, 480, 522, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[551, 526, 479, 487, 520, 495, 496, 549, 516, 509, 516, 514, 491, 525, 530, 502, 571, 494, 495, 506, 510, 489, 510, 537, 509, 523, 491, 469, 513, 533, 526, 513, 508, 552, 506, 467, 534, 529, 529, 478, 497, 527, 517, 526, 548, 518, 499, 515, 537, 523, 501, 480, 488, 484, 541, 534, 514, 516, 491, 468, 529, 492, 497, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[497, 482, 533, 515, 500, 514, 525, 538, 576, 500, 511, 512, 481, 520, 462, 521, 480, 504, 549, 511, 500, 523, 519, 525, 534, 487, 539, 502, 525, 519, 526, 525, 527, 532, 542, 547, 461, 526, 539, 466, 488, 517, 545, 530, 478, 515, 476, 446, 490, 515, 536, 496, 536, 529, 489, 500, 521, 501, 537, 515, 512, 509, 485, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[530, 503, 498, 534, 532, 523, 518, 509, 544, 505, 499, 509, 517, 516, 498, 473, 536, 486, 540, 493, 482, 501, 512, 502, 517, 509, 517, 552, 520, 480, 509, 505, 524, 507, 495, 529, 486, 512, 466, 495, 555, 500, 541, 508, 490, 529, 503, 501, 511, 502, 477, 493, 520, 533, 513, 514, 491, 495, 543, 534, 543, 549, 518, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[494, 498, 478, 499, 467, 501, 459, 519, 527, 523, 548, 477, 540, 526, 516, 507, 510, 539, 532, 527, 525, 484, 520, 502, 481, 542, 476, 494, 485, 516, 536, 522, 509, 495, 527, 508, 513, 481, 527, 484, 524, 522, 566, 527, 493, 522, 512, 522, 527, 529, 531, 489, 529, 578, 488, 481, 536, 543, 501, 521, 513, 517, 507, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[521, 503, 529, 480, 500, 467, 505, 505, 500, 529, 531, 515, 488, 530, 543, 499, 499, 513, 524, 506, 527, 517, 492, 530, 554, 545, 507, 498, 498, 549, 520, 523, 544, 454, 477, 506, 487, 504, 517, 481, 518, 508, 504, 521, 524, 530, 470, 519, 501, 495, 492, 527, 513, 491, 496, 555, 541, 520, 529, 491, 527, 561, 491, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[484, 494, 499, 542, 508, 495, 528, 527, 511, 485, 515, 515, 521, 495, 535, 484, 526, 545, 543, 522, 506, 517, 533, 501, 518, 518, 516, 501, 514, 535, 532, 511, 505, 512, 519, 515, 512, 520, 527, 522, 517, 518, 517, 534, 483, 525, 459, 490, 523, 515, 490, 503, 536, 501, 509, 484, 503, 519, 509, 437, 487, 529, 550, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[528, 502, 507, 529, 479, 507, 478, 484, 518, 499, 544, 492, 509, 511, 547, 472, 532, 543, 476, 498, 472, 505, 518, 496, 513, 521, 535, 506, 515, 466, 493, 521, 526, 539, 519, 519, 495, 507, 497, 532, 528, 534, 528, 511, 488, 527, 498, 535, 526, 471, 530, 522, 514, 514, 508, 547, 500, 511, 502, 497, 515, 528, 536, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 500, 533, 535, 517, 530, 485, 521, 20, 12, 0, 0, 0, 0, 0, 0, 489, 504, 488, 534, 514, 482, 468, 512, 8, 1, 0, 0, 0, 0, 0, 0, 511, 517, 511, 523, 521, 541, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 516, 547, 555, 508, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 529, 532, 543, 546, 539, 508, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 485, 511, 507, 472, 559, 459, 523, 483, 2, 0, 0, 0, 0, 0, 0, 0, 522, 485, 523, 521, 503, 495, 508, 509, 5, 0, 0, 0, 0, 0, 0, 0, 537, 474, 496, 492, 517, 472, 522, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4000, 4158, 4108, 4226, 4001, 4071, 4040] avg=4096.0000 max=4226 max_rank=4 min=4000 min_rank=1 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[551, 526, 479, 487, 520, 495, 496, 549, 19, 12, 0, 0, 0, 0, 0, 0, 516, 509, 516, 514, 491, 525, 530, 502, 7, 1, 0, 0, 0, 0, 0, 0, 571, 494, 476, 506, 510, 489, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 509, 523, 491, 469, 513, 533, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 508, 552, 506, 467, 534, 529, 529, 478, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 517, 526, 548, 518, 499, 515, 3, 0, 0, 0, 0, 0, 0, 0, 537, 523, 501, 480, 488, 484, 541, 534, 5, 0, 0, 0, 0, 0, 0, 0, 514, 516, 491, 468, 529, 483, 497, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4111, 4093, 4058, 4103, 4150, 4093, 4026] avg=4096.0000 max=4150 max_rank=5 min=4026 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 503, 498, 534, 532, 523, 518, 509, 21, 12, 0, 0, 0, 0, 0, 0, 544, 505, 499, 509, 517, 516, 498, 473, 7, 1, 0, 0, 0, 0, 0, 0, 536, 486, 519, 493, 482, 501, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 517, 552, 520, 480, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 495, 529, 486, 512, 466, 495, 0, 0, 0, 0, 0, 0, 0, 0, 555, 500, 541, 508, 490, 529, 503, 501, 3, 0, 0, 0, 0, 0, 0, 0, 511, 502, 477, 493, 520, 533, 513, 514, 5, 0, 0, 0, 0, 0, 0, 0, 491, 495, 543, 534, 543, 540, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4069, 4031, 4090, 4014, 4130, 4068, 4186] avg=4096.0000 max=4186 max_rank=7 min=4014 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 482, 533, 515, 500, 514, 525, 538, 21, 12, 0, 0, 0, 0, 0, 0, 576, 500, 511, 512, 481, 520, 462, 521, 7, 1, 0, 0, 0, 0, 0, 0, 480, 504, 528, 511, 500, 523, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 539, 502, 525, 519, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 532, 542, 547, 461, 526, 539, 466, 0, 0, 0, 0, 0, 0, 0, 0, 488, 517, 545, 530, 478, 515, 476, 446, 3, 0, 0, 0, 0, 0, 0, 0, 490, 515, 536, 496, 536, 529, 489, 500, 5, 0, 0, 0, 0, 0, 0, 0, 521, 501, 537, 515, 512, 500, 485, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4091, 4090, 4138, 4140, 3998, 4096, 4078] avg=4096.0000 max=4140 max_rank=4 min=3998 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[521, 503, 529, 480, 500, 467, 505, 505, 20, 12, 0, 0, 0, 0, 0, 0, 500, 529, 531, 515, 488, 530, 543, 499, 7, 2, 0, 0, 0, 0, 0, 0, 499, 513, 504, 506, 527, 517, 492, 530, 0, 0, 0, 0, 0, 0, 0, 0, 554, 545, 507, 498, 498, 549, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 544, 454, 477, 506, 487, 504, 517, 481, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 504, 521, 524, 530, 470, 519, 3, 0, 0, 0, 0, 0, 0, 0, 501, 495, 492, 527, 513, 491, 496, 555, 6, 0, 0, 0, 0, 0, 0, 0, 541, 520, 529, 491, 527, 550, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4144, 4088, 4175, 3970, 4097, 4076, 4176] avg=4096.0000 max=4176 max_rank=7 min=3970 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[494, 498, 478, 499, 467, 501, 459, 519, 20, 12, 0, 0, 0, 0, 0, 0, 527, 523, 548, 477, 540, 526, 516, 507, 8, 1, 0, 0, 0, 0, 0, 0, 510, 539, 512, 527, 525, 484, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 481, 542, 476, 494, 485, 516, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 527, 508, 513, 481, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 566, 527, 493, 522, 512, 522, 3, 0, 0, 0, 0, 0, 0, 0, 527, 529, 531, 489, 529, 578, 488, 481, 5, 0, 0, 0, 0, 0, 0, 0, 536, 543, 501, 521, 513, 508, 507, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3947, 4173, 4119, 4032, 4044, 4191, 4157, 4105] avg=4096.0000 max=4191 max_rank=5 min=3947 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 494, 499, 542, 508, 495, 528, 527, 26, 18, 0, 0, 0, 0, 0, 0, 511, 485, 515, 515, 521, 495, 535, 484, 12, 7, 0, 0, 0, 0, 0, 0, 526, 545, 517, 522, 506, 517, 533, 501, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 516, 501, 514, 535, 502, 511, 0, 0, 0, 0, 0, 0, 0, 0, 505, 512, 519, 515, 512, 520, 527, 522, 5, 0, 0, 0, 0, 0, 0, 0, 517, 518, 517, 534, 483, 525, 459, 490, 6, 0, 0, 0, 0, 0, 0, 0, 523, 515, 490, 503, 536, 501, 509, 484, 9, 0, 0, 0, 0, 0, 0, 0, 503, 519, 509, 437, 487, 502, 550, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4080, 4167, 4115, 4137, 4049, 4070, 4029] avg=4096.0000 max=4167 max_rank=2 min=4029 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[528, 502, 507, 529, 479, 507, 478, 484, 18, 11, 0, 0, 0, 0, 0, 0, 518, 499, 544, 492, 509, 511, 547, 472, 7, 1, 0, 0, 0, 0, 0, 0, 532, 543, 458, 498, 472, 505, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 535, 506, 515, 466, 475, 521, 0, 0, 0, 0, 0, 0, 0, 0, 526, 539, 519, 519, 495, 507, 497, 532, 0, 0, 0, 0, 0, 0, 0, 0, 528, 534, 528, 511, 488, 527, 498, 535, 3, 0, 0, 0, 0, 0, 0, 0, 526, 471, 530, 522, 514, 514, 508, 547, 5, 0, 0, 0, 0, 0, 0, 0, 500, 511, 502, 497, 515, 519, 536, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4100, 4022, 4052, 4134, 4152, 4137, 4128] avg=4096.0000 max=4152 max_rank=5 min=4022 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4352, 4224, 4096, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[540, 506, 516, 542, 522, 526, 519, 515, 510, 497, 519, 493, 453, 535, 526, 509, 501, 450, 533, 514, 499, 502, 516, 502, 518, 522, 512, 486, 537, 529, 489, 535, 515, 509, 517, 501, 482, 533, 566, 480, 514, 514, 540, 477, 501, 516, 484, 543, 513, 493, 494, 526, 514, 523, 542, 475, 524, 497, 513, 482, 560, 531, 507, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[478, 450, 530, 489, 516, 499, 495, 529, 532, 505, 545, 505, 510, 519, 513, 465, 485, 539, 519, 528, 505, 506, 465, 479, 510, 535, 514, 556, 537, 487, 476, 515, 549, 514, 496, 519, 501, 509, 566, 538, 536, 508, 490, 498, 505, 529, 534, 517, 524, 484, 551, 511, 492, 530, 539, 510, 530, 504, 514, 499, 498, 503, 540, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[508, 551, 533, 500, 519, 500, 526, 508, 532, 500, 487, 547, 521, 500, 517, 525, 541, 492, 497, 545, 540, 490, 485, 494, 503, 527, 468, 498, 498, 486, 488, 528, 497, 522, 513, 536, 530, 503, 510, 541, 500, 513, 540, 529, 529, 498, 502, 542, 530, 505, 513, 497, 503, 466, 523, 528, 477, 488, 481, 543, 511, 506, 515, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[495, 525, 494, 489, 509, 521, 513, 537, 482, 537, 527, 530, 502, 535, 493, 531, 484, 487, 527, 532, 522, 518, 529, 519, 493, 517, 502, 498, 529, 487, 522, 538, 524, 511, 508, 507, 508, 500, 497, 476, 541, 549, 512, 501, 492, 537, 500, 527, 449, 543, 513, 514, 468, 524, 494, 524, 516, 505, 501, 491, 553, 532, 496, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[525, 498, 504, 528, 500, 501, 529, 476, 505, 539, 515, 540, 502, 494, 510, 492, 509, 508, 515, 516, 496, 483, 517, 538, 506, 511, 476, 521, 547, 535, 551, 517, 519, 492, 522, 475, 548, 472, 539, 520, 523, 508, 546, 523, 489, 528, 480, 497, 526, 493, 504, 551, 490, 525, 502, 546, 485, 520, 514, 513, 499, 504, 503, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[518, 490, 517, 479, 533, 513, 493, 541, 539, 507, 497, 502, 501, 524, 543, 483, 535, 490, 498, 484, 476, 504, 573, 531, 521, 526, 517, 508, 511, 550, 493, 558, 511, 501, 484, 533, 524, 497, 514, 544, 503, 486, 482, 461, 519, 522, 491, 482, 492, 504, 497, 501, 534, 491, 555, 514, 510, 562, 509, 527, 524, 529, 491, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[493, 546, 562, 533, 521, 474, 549, 520, 530, 516, 506, 528, 521, 485, 486, 561, 495, 535, 518, 499, 525, 468, 517, 508, 498, 457, 482, 518, 517, 511, 492, 516, 518, 547, 514, 535, 487, 510, 508, 508, 518, 536, 518, 536, 507, 511, 498, 498, 504, 497, 512, 546, 498, 480, 511, 522, 537, 472, 492, 494, 506, 512, 532, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[491, 520, 527, 527, 498, 468, 492, 516, 530, 500, 522, 518, 482, 550, 540, 513, 501, 530, 494, 493, 508, 519, 494, 520, 476, 486, 515, 487, 534, 487, 526, 489, 521, 533, 519, 503, 513, 524, 485, 522, 520, 551, 537, 481, 520, 519, 486, 488, 502, 513, 499, 531, 537, 541, 516, 548, 516, 516, 547, 484, 519, 502, 540, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[478, 450, 529, 489, 516, 499, 495, 529, 0, 0, 0, 0, 0, 0, 0, 0, 532, 505, 545, 493, 510, 519, 513, 465, 0, 0, 0, 0, 0, 0, 0, 0, 485, 539, 519, 528, 505, 506, 465, 479, 20, 8, 0, 0, 0, 0, 0, 0, 510, 535, 514, 556, 537, 487, 476, 515, 4, 3, 1, 0, 0, 0, 0, 0, 549, 514, 496, 519, 501, 509, 546, 538, 0, 0, 0, 0, 0, 0, 0, 0, 536, 507, 490, 498, 505, 529, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 551, 511, 492, 530, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 514, 499, 498, 503, 540, 494, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4082, 4054, 4138, 4172, 4116, 4138, 4083] avg=4096.0000 max=4172 max_rank=4 min=3985 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 551, 532, 500, 519, 500, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 532, 500, 487, 533, 521, 500, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 541, 492, 497, 545, 540, 490, 485, 494, 18, 9, 0, 0, 0, 0, 0, 0, 503, 527, 468, 498, 498, 486, 488, 528, 5, 3, 1, 0, 0, 0, 0, 0, 497, 522, 513, 536, 530, 503, 492, 541, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 540, 529, 529, 498, 502, 542, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 513, 497, 503, 466, 520, 528, 0, 0, 0, 0, 0, 0, 0, 0, 477, 488, 481, 543, 511, 506, 515, 523, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4115, 4111, 4005, 4134, 4152, 4062, 4045] avg=4096.0000 max=4152 max_rank=5 min=4005 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[540, 506, 515, 542, 522, 526, 519, 515, 0, 0, 0, 0, 0, 0, 0, 0, 510, 497, 519, 481, 453, 535, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 501, 450, 533, 514, 499, 502, 516, 502, 20, 8, 0, 0, 0, 0, 0, 0, 518, 522, 512, 486, 537, 529, 489, 535, 4, 3, 1, 0, 0, 0, 0, 0, 515, 509, 517, 501, 482, 533, 546, 480, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 540, 477, 501, 516, 484, 543, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 494, 526, 514, 523, 539, 475, 0, 0, 0, 0, 0, 0, 0, 0, 524, 497, 513, 482, 560, 531, 507, 509, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4030, 4045, 4136, 4083, 4088, 4077, 4124] avg=4096.0000 max=4185 max_rank=0 min=4030 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[495, 525, 493, 489, 509, 521, 513, 537, 0, 0, 0, 0, 0, 0, 0, 0, 482, 537, 527, 516, 502, 535, 493, 531, 0, 0, 0, 0, 0, 0, 0, 0, 484, 487, 527, 532, 522, 518, 529, 519, 18, 9, 0, 0, 0, 0, 0, 0, 493, 517, 502, 498, 529, 487, 522, 538, 5, 3, 1, 0, 0, 0, 0, 0, 524, 511, 508, 507, 508, 500, 479, 476, 0, 0, 0, 0, 0, 0, 0, 0, 541, 548, 512, 501, 492, 537, 500, 527, 0, 0, 0, 0, 0, 0, 0, 0, 449, 543, 513, 514, 468, 524, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 501, 491, 553, 532, 496, 531, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4123, 4145, 4095, 4013, 4158, 4026, 4126] avg=4096.0000 max=4158 max_rank=5 min=4013 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 498, 503, 528, 500, 501, 529, 476, 0, 0, 0, 0, 0, 0, 0, 0, 505, 539, 515, 526, 502, 494, 510, 492, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 515, 516, 496, 483, 517, 538, 19, 9, 0, 0, 0, 0, 0, 0, 506, 511, 476, 521, 547, 535, 551, 517, 5, 3, 1, 0, 0, 0, 0, 0, 519, 492, 522, 475, 548, 472, 520, 520, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 546, 523, 489, 528, 480, 497, 0, 0, 0, 0, 0, 0, 0, 0, 526, 493, 504, 551, 490, 525, 499, 546, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 514, 513, 499, 504, 503, 508, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4083, 4110, 4173, 4068, 4093, 4134, 4047] avg=4096.0000 max=4173 max_rank=3 min=4047 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 516, 479, 533, 513, 493, 541, 0, 0, 0, 0, 0, 0, 0, 0, 539, 507, 497, 490, 501, 524, 543, 483, 0, 0, 0, 0, 0, 0, 0, 0, 535, 490, 498, 484, 476, 504, 573, 531, 18, 8, 0, 0, 0, 0, 0, 0, 521, 526, 517, 508, 511, 550, 493, 558, 4, 3, 1, 0, 0, 0, 0, 0, 511, 501, 484, 533, 524, 497, 496, 544, 0, 0, 0, 0, 0, 0, 0, 0, 503, 485, 482, 461, 519, 522, 491, 482, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 497, 501, 534, 491, 552, 514, 0, 0, 0, 0, 0, 0, 0, 0, 510, 562, 509, 527, 524, 529, 491, 519, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4084, 4117, 4192, 4090, 3945, 4085, 4172] avg=4096.0000 max=4192 max_rank=3 min=3945 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 546, 555, 533, 521, 474, 549, 520, 0, 0, 0, 0, 0, 0, 0, 0, 530, 516, 506, 505, 521, 485, 486, 561, 0, 0, 0, 0, 0, 0, 0, 0, 495, 535, 518, 499, 525, 468, 517, 508, 22, 14, 0, 0, 0, 0, 0, 0, 498, 457, 482, 518, 517, 511, 492, 516, 9, 8, 7, 5, 0, 0, 0, 0, 518, 547, 514, 535, 487, 510, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 518, 536, 507, 511, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 512, 546, 498, 480, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 537, 472, 492, 494, 506, 512, 532, 517, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4110, 4101, 4020, 4105, 4115, 4062, 4064] avg=4096.0000 max=4191 max_rank=0 min=4020 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 520, 526, 527, 498, 468, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 522, 504, 482, 550, 540, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 494, 493, 508, 519, 494, 520, 17, 9, 0, 0, 0, 0, 0, 0, 476, 486, 515, 487, 534, 487, 526, 489, 5, 3, 1, 0, 0, 0, 0, 0, 521, 533, 519, 503, 513, 524, 468, 522, 0, 0, 0, 0, 0, 0, 0, 0, 520, 550, 537, 481, 520, 519, 486, 488, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 499, 531, 537, 541, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 547, 484, 519, 502, 540, 482, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4141, 4085, 4009, 4103, 4101, 4184, 4107] avg=4096.0000 max=4184 max_rank=6 min=4009 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[510, 511, 515, 536, 515, 502, 534, 515, 523, 477, 535, 523, 487, 510, 513, 516, 519, 537, 525, 491, 514, 482, 555, 553, 502, 522, 531, 505, 515, 511, 499, 516, 529, 492, 449, 508, 462, 527, 526, 509, 527, 459, 481, 525, 511, 523, 535, 516, 499, 528, 493, 504, 522, 509, 553, 528, 533, 512, 511, 489, 533, 487, 507, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[498, 482, 522, 516, 510, 541, 500, 496, 540, 528, 501, 505, 509, 528, 535, 494, 502, 521, 538, 480, 503, 543, 518, 537, 470, 523, 524, 508, 516, 498, 488, 522, 496, 493, 471, 558, 550, 524, 545, 511, 485, 512, 511, 513, 508, 486, 488, 493, 482, 508, 524, 477, 527, 536, 513, 503, 456, 548, 523, 513, 540, 515, 534, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[508, 504, 547, 521, 526, 508, 535, 540, 487, 524, 538, 532, 493, 525, 579, 555, 518, 449, 498, 540, 540, 499, 523, 540, 502, 474, 498, 478, 545, 518, 489, 504, 537, 505, 485, 514, 486, 494, 536, 507, 520, 506, 503, 523, 517, 546, 483, 510, 470, 507, 542, 515, 499, 514, 530, 500, 499, 475, 512, 483, 504, 514, 502, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[502, 536, 535, 487, 492, 522, 497, 510, 508, 494, 546, 535, 546, 505, 496, 491, 533, 492, 510, 469, 505, 506, 496, 495, 540, 488, 528, 513, 500, 491, 512, 555, 505, 483, 496, 522, 517, 542, 523, 523, 558, 517, 529, 511, 523, 505, 517, 501, 541, 480, 491, 531, 509, 506, 501, 496, 532, 482, 525, 520, 514, 513, 494, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[490, 532, 531, 504, 552, 498, 514, 523, 529, 523, 507, 522, 518, 522, 538, 486, 524, 547, 521, 484, 517, 516, 537, 537, 524, 480, 514, 483, 509, 493, 473, 490, 528, 505, 471, 514, 463, 525, 490, 452, 503, 563, 545, 483, 496, 508, 513, 495, 499, 521, 522, 528, 507, 495, 489, 523, 509, 529, 488, 568, 535, 509, 528, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[520, 526, 506, 523, 503, 531, 521, 496, 477, 485, 561, 499, 526, 498, 558, 520, 545, 490, 512, 528, 496, 508, 535, 503, 481, 527, 524, 506, 504, 510, 509, 483, 539, 478, 511, 500, 481, 541, 507, 531, 525, 523, 510, 514, 551, 520, 481, 487, 499, 503, 518, 489, 490, 479, 547, 504, 499, 557, 470, 530, 498, 517, 517, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[530, 521, 494, 513, 483, 541, 523, 515, 535, 533, 487, 496, 497, 513, 507, 512, 547, 504, 512, 524, 553, 489, 566, 481, 520, 467, 539, 504, 516, 498, 528, 511, 488, 548, 561, 503, 487, 484, 513, 491, 519, 521, 511, 520, 506, 493, 477, 515, 500, 522, 514, 513, 550, 503, 528, 485, 483, 520, 520, 486, 523, 540, 475, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[512, 535, 505, 483, 536, 504, 498, 535, 528, 514, 487, 528, 511, 502, 520, 501, 534, 500, 496, 532, 511, 519, 488, 479, 493, 485, 529, 485, 541, 530, 499, 513, 520, 479, 541, 507, 546, 495, 499, 495, 491, 543, 533, 545, 507, 506, 525, 522, 537, 510, 457, 534, 524, 507, 487, 512, 533, 528, 516, 479, 515, 528, 520, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[498, 482, 497, 516, 510, 541, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 540, 528, 501, 505, 509, 528, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 479, 521, 538, 480, 503, 543, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 470, 523, 524, 508, 516, 498, 488, 522, 31, 8, 0, 0, 0, 0, 0, 0, 496, 493, 471, 558, 550, 524, 545, 511, 17, 13, 0, 0, 0, 0, 0, 0, 485, 509, 511, 513, 508, 486, 488, 493, 0, 0, 0, 0, 0, 0, 0, 0, 482, 508, 524, 477, 527, 536, 513, 503, 10, 3, 2, 0, 0, 0, 0, 0, 456, 548, 523, 513, 538, 515, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4109, 4119, 4088, 4178, 3993, 4085, 4156] avg=4096.0000 max=4178 max_rank=4 min=3993 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 511, 490, 536, 515, 502, 534, 515, 0, 0, 0, 0, 0, 0, 0, 0, 523, 477, 535, 523, 487, 510, 483, 516, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 525, 491, 514, 482, 555, 553, 0, 0, 0, 0, 0, 0, 0, 0, 502, 522, 531, 505, 515, 511, 499, 516, 30, 8, 0, 0, 0, 0, 0, 0, 529, 492, 449, 508, 462, 527, 526, 509, 17, 13, 0, 0, 0, 0, 0, 0, 527, 457, 481, 525, 511, 523, 535, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 528, 493, 504, 522, 509, 553, 528, 10, 2, 2, 0, 0, 0, 0, 0, 533, 512, 511, 489, 531, 487, 507, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4054, 4153, 4139, 4032, 4075, 4150, 4052] avg=4096.0000 max=4153 max_rank=2 min=4032 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[502, 536, 510, 487, 492, 522, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 546, 535, 546, 505, 467, 491, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 510, 469, 505, 506, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 540, 488, 528, 513, 500, 491, 512, 555, 29, 8, 0, 0, 0, 0, 0, 0, 505, 483, 496, 522, 517, 542, 523, 523, 17, 14, 0, 0, 0, 0, 0, 0, 558, 514, 529, 511, 523, 505, 517, 501, 0, 0, 0, 0, 0, 0, 0, 0, 541, 480, 491, 531, 509, 506, 501, 496, 10, 3, 2, 0, 0, 0, 0, 0, 532, 482, 525, 520, 512, 513, 494, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4092, 3982, 4164, 4142, 4158, 4070, 4104] avg=4096.0000 max=4164 max_rank=3 min=3982 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[508, 504, 521, 521, 526, 508, 535, 540, 0, 0, 0, 0, 0, 0, 0, 0, 487, 524, 538, 532, 493, 525, 545, 555, 0, 0, 0, 0, 0, 0, 0, 0, 495, 449, 498, 540, 540, 499, 523, 540, 0, 0, 0, 0, 0, 0, 0, 0, 502, 474, 498, 478, 545, 518, 489, 504, 34, 8, 0, 0, 0, 0, 0, 0, 537, 505, 485, 514, 486, 494, 536, 507, 18, 13, 0, 0, 0, 0, 0, 0, 520, 503, 503, 523, 517, 546, 483, 510, 0, 0, 0, 0, 0, 0, 0, 0, 470, 507, 542, 515, 499, 514, 530, 500, 10, 3, 2, 0, 0, 0, 0, 0, 499, 475, 512, 483, 502, 514, 502, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4199, 4084, 4050, 4095, 4105, 4092, 3980] avg=4096.0000 max=4199 max_rank=1 min=3980 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 526, 483, 523, 503, 531, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 477, 485, 561, 499, 526, 498, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 490, 512, 528, 496, 508, 535, 503, 0, 0, 0, 0, 0, 0, 0, 0, 481, 527, 524, 506, 504, 510, 509, 483, 32, 7, 0, 0, 0, 0, 0, 0, 539, 478, 511, 500, 481, 541, 507, 531, 16, 14, 0, 0, 0, 0, 0, 0, 525, 520, 510, 514, 551, 520, 481, 487, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 518, 489, 490, 479, 547, 504, 11, 3, 2, 0, 0, 0, 0, 0, 499, 557, 470, 530, 496, 517, 517, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4092, 4092, 4083, 4118, 4108, 4045, 4127] avg=4096.0000 max=4127 max_rank=7 min=4045 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[490, 532, 506, 504, 552, 498, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 507, 522, 518, 522, 507, 486, 0, 0, 0, 0, 0, 0, 0, 0, 501, 547, 521, 484, 517, 516, 537, 537, 0, 0, 0, 0, 0, 0, 0, 0, 524, 480, 514, 483, 509, 493, 473, 490, 31, 8, 0, 0, 0, 0, 0, 0, 528, 505, 471, 514, 463, 525, 490, 452, 17, 13, 0, 0, 0, 0, 0, 0, 503, 560, 545, 483, 496, 508, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 522, 528, 507, 495, 489, 523, 10, 3, 2, 0, 0, 0, 0, 0, 509, 529, 488, 568, 533, 509, 528, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4114, 4160, 4005, 3978, 4103, 4099, 4190] avg=4096.0000 max=4190 max_rank=7 min=3978 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[530, 521, 471, 513, 483, 541, 523, 515, 0, 0, 0, 0, 0, 0, 0, 0, 535, 533, 487, 496, 497, 513, 478, 512, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 512, 524, 553, 489, 566, 481, 0, 0, 0, 0, 0, 0, 0, 0, 520, 467, 539, 504, 516, 498, 528, 511, 29, 7, 0, 0, 0, 0, 0, 0, 488, 548, 561, 503, 487, 484, 513, 491, 16, 14, 0, 0, 0, 0, 0, 0, 519, 518, 511, 520, 506, 493, 477, 515, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 514, 513, 550, 503, 528, 485, 11, 3, 2, 0, 0, 0, 0, 0, 483, 520, 520, 486, 521, 540, 475, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4051, 4151, 4119, 4105, 4059, 4131, 4055] avg=4096.0000 max=4151 max_rank=2 min=4051 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 535, 474, 483, 536, 504, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 487, 528, 511, 502, 486, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 496, 532, 511, 519, 488, 479, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 529, 485, 541, 530, 499, 513, 34, 11, 0, 0, 0, 0, 0, 0, 520, 479, 541, 507, 546, 495, 499, 495, 20, 18, 0, 0, 0, 0, 0, 0, 491, 538, 533, 545, 507, 506, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 457, 534, 524, 507, 487, 512, 14, 5, 9, 0, 0, 0, 0, 0, 533, 528, 516, 479, 506, 528, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4057, 4027, 4120, 4120, 4167, 4096, 4104] avg=4096.0000 max=4167 max_rank=5 min=4027 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[517, 519, 519, 556, 471, 495, 543, 527, 486, 506, 550, 487, 484, 514, 475, 522, 517, 490, 557, 497, 486, 503, 504, 466, 529, 505, 492, 502, 526, 500, 513, 496, 514, 534, 544, 517, 543, 531, 505, 523, 543, 486, 495, 558, 527, 502, 493, 510, 529, 531, 499, 487, 497, 537, 544, 499, 514, 517, 511, 537, 491, 492, 507, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[511, 552, 532, 489, 514, 521, 488, 513, 506, 525, 542, 494, 489, 514, 514, 511, 547, 504, 524, 542, 544, 499, 484, 516, 516, 483, 486, 529, 511, 499, 521, 476, 522, 496, 526, 515, 500, 517, 531, 537, 498, 504, 522, 517, 507, 512, 511, 467, 537, 494, 512, 523, 502, 519, 582, 519, 527, 490, 498, 468, 513, 502, 511, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[526, 521, 481, 510, 508, 475, 520, 552, 509, 491, 517, 521, 544, 496, 563, 497, 506, 507, 497, 508, 486, 520, 511, 553, 504, 506, 475, 510, 528, 504, 520, 506, 500, 513, 501, 514, 508, 522, 455, 507, 518, 535, 529, 532, 517, 540, 556, 517, 480, 518, 540, 502, 484, 497, 488, 541, 529, 488, 506, 524, 510, 490, 515, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[487, 512, 487, 532, 503, 486, 513, 511, 509, 521, 481, 511, 506, 529, 516, 528, 536, 546, 504, 521, 525, 490, 515, 513, 513, 505, 489, 511, 495, 473, 518, 537, 513, 475, 495, 495, 487, 512, 533, 516, 512, 515, 520, 519, 465, 527, 527, 507, 530, 526, 514, 482, 539, 540, 511, 529, 531, 491, 502, 525, 535, 539, 530, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[505, 505, 505, 504, 558, 542, 509, 562, 513, 500, 491, 486, 492, 511, 492, 481, 477, 531, 548, 515, 506, 487, 487, 512, 517, 538, 522, 504, 484, 520, 483, 524, 511, 528, 525, 505, 533, 517, 516, 461, 493, 523, 499, 529, 537, 528, 541, 472, 497, 498, 503, 468, 506, 498, 527, 529, 552, 555, 472, 545, 538, 516, 500, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[515, 486, 525, 565, 511, 504, 505, 553, 543, 538, 512, 513, 510, 501, 489, 523, 548, 507, 486, 514, 516, 477, 517, 530, 525, 528, 509, 539, 512, 472, 500, 501, 529, 481, 540, 511, 511, 542, 503, 519, 483, 492, 521, 508, 513, 476, 495, 555, 498, 484, 518, 518, 480, 487, 516, 516, 540, 515, 502, 478, 546, 520, 527, 470] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[545, 531, 501, 503, 538, 502, 505, 504, 501, 484, 556, 505, 471, 538, 533, 485, 535, 517, 509, 482, 511, 496, 513, 506, 490, 507, 516, 534, 520, 524, 501, 528, 555, 520, 503, 492, 517, 550, 486, 503, 460, 517, 529, 541, 525, 511, 468, 538, 508, 491, 487, 517, 503, 561, 504, 510, 517, 499, 504, 501, 517, 491, 542, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[455, 535, 490, 541, 515, 525, 516, 503, 516, 510, 515, 525, 548, 513, 516, 528, 537, 542, 490, 490, 514, 511, 519, 452, 543, 504, 518, 505, 543, 537, 517, 523, 517, 498, 509, 489, 476, 504, 534, 471, 534, 506, 517, 501, 528, 535, 505, 543, 531, 530, 500, 511, 506, 477, 539, 503, 526, 468, 508, 491, 492, 480, 509, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 519, 519, 556, 471, 495, 543, 506, 0, 0, 0, 0, 0, 0, 0, 0, 486, 506, 550, 487, 484, 514, 475, 522, 8, 0, 0, 0, 0, 0, 0, 0, 517, 490, 557, 497, 486, 503, 504, 466, 7, 1, 0, 0, 0, 0, 0, 0, 529, 505, 492, 502, 526, 500, 513, 496, 12, 0, 0, 0, 0, 0, 0, 0, 514, 534, 544, 517, 543, 531, 505, 523, 1, 0, 0, 0, 0, 0, 0, 0, 543, 486, 495, 541, 527, 502, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 499, 487, 497, 537, 544, 499, 1, 0, 0, 0, 0, 0, 0, 0, 514, 517, 511, 537, 491, 492, 507, 497, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4032, 4028, 4075, 4212, 4097, 4124, 4074] avg=4096.0000 max=4212 max_rank=4 min=4028 min_rank=2 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[505, 505, 505, 504, 558, 542, 509, 539, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 491, 486, 492, 511, 492, 481, 9, 0, 0, 0, 0, 0, 0, 0, 477, 531, 548, 515, 506, 487, 487, 512, 6, 1, 0, 0, 0, 0, 0, 0, 517, 538, 522, 504, 484, 520, 483, 524, 13, 0, 0, 0, 0, 0, 0, 0, 511, 528, 525, 505, 533, 517, 516, 461, 1, 0, 0, 0, 0, 0, 0, 0, 493, 523, 499, 514, 537, 528, 541, 472, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 503, 468, 506, 498, 527, 529, 1, 0, 0, 0, 0, 0, 0, 0, 552, 555, 472, 545, 538, 516, 500, 535, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 3975, 4070, 4105, 4097, 4107, 4027, 4220] avg=4096.0000 max=4220 max_rank=7 min=3975 min_rank=1 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 552, 532, 489, 514, 521, 488, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 542, 494, 489, 514, 514, 511, 8, 0, 0, 0, 0, 0, 0, 0, 547, 504, 524, 542, 544, 499, 484, 516, 6, 1, 0, 0, 0, 0, 0, 0, 516, 483, 486, 529, 511, 499, 521, 476, 12, 0, 0, 0, 0, 0, 0, 0, 522, 496, 526, 515, 500, 517, 531, 537, 1, 0, 0, 0, 0, 0, 0, 0, 498, 504, 522, 502, 507, 512, 511, 467, 0, 0, 0, 0, 0, 0, 0, 0, 537, 494, 512, 523, 502, 519, 582, 519, 1, 0, 0, 0, 0, 0, 0, 0, 527, 490, 498, 468, 513, 502, 511, 493, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4103, 4167, 4033, 4145, 4023, 4189, 4009] avg=4096.0000 max=4189 max_rank=6 min=4009 min_rank=7 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 521, 481, 510, 508, 475, 520, 529, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 517, 521, 544, 496, 563, 497, 9, 0, 0, 0, 0, 0, 0, 0, 506, 507, 497, 508, 486, 520, 511, 553, 6, 1, 0, 0, 0, 0, 0, 0, 504, 506, 475, 510, 528, 504, 520, 506, 13, 0, 0, 0, 0, 0, 0, 0, 500, 513, 501, 514, 508, 522, 455, 507, 1, 0, 0, 0, 0, 0, 0, 0, 518, 535, 529, 517, 517, 540, 556, 517, 0, 0, 0, 0, 0, 0, 0, 0, 480, 518, 540, 502, 484, 497, 488, 541, 1, 0, 0, 0, 0, 0, 0, 0, 529, 488, 506, 524, 510, 490, 515, 520, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4147, 4095, 4066, 4021, 4229, 4051, 4089] avg=4096.0000 max=4229 max_rank=5 min=4021 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 486, 525, 565, 511, 504, 505, 530, 0, 0, 0, 0, 0, 0, 0, 0, 543, 538, 512, 513, 510, 501, 489, 523, 9, 0, 0, 0, 0, 0, 0, 0, 548, 507, 486, 514, 516, 477, 517, 530, 6, 1, 0, 0, 0, 0, 0, 0, 525, 528, 509, 539, 512, 472, 500, 501, 13, 0, 0, 0, 0, 0, 0, 0, 529, 481, 540, 511, 511, 542, 503, 519, 1, 0, 0, 0, 0, 0, 0, 0, 483, 492, 521, 493, 513, 476, 495, 555, 0, 0, 0, 0, 0, 0, 0, 0, 498, 484, 518, 518, 480, 487, 516, 516, 1, 0, 0, 0, 0, 0, 0, 0, 540, 515, 502, 478, 546, 520, 527, 470, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4138, 4102, 4099, 4137, 4028, 4018, 4105] avg=4096.0000 max=4141 max_rank=0 min=4018 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 512, 487, 532, 503, 486, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 481, 511, 506, 529, 516, 528, 8, 0, 0, 0, 0, 0, 0, 0, 536, 546, 504, 521, 525, 490, 515, 513, 6, 1, 0, 0, 0, 0, 0, 0, 513, 505, 489, 511, 495, 473, 518, 537, 12, 0, 0, 0, 0, 0, 0, 0, 513, 475, 495, 495, 487, 512, 533, 516, 1, 0, 0, 0, 0, 0, 0, 0, 512, 515, 520, 504, 465, 527, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 530, 526, 514, 482, 539, 540, 511, 529, 1, 0, 0, 0, 0, 0, 0, 0, 531, 491, 502, 525, 535, 539, 530, 503, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4109, 4157, 4053, 4027, 4077, 4172, 4163] avg=4096.0000 max=4172 max_rank=6 min=4010 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[455, 535, 490, 541, 515, 525, 516, 482, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 515, 525, 548, 513, 516, 528, 8, 0, 0, 0, 0, 0, 0, 0, 537, 542, 490, 490, 514, 511, 519, 452, 6, 1, 0, 0, 0, 0, 0, 0, 543, 504, 518, 505, 543, 537, 517, 523, 12, 0, 0, 0, 0, 0, 0, 0, 517, 498, 509, 489, 476, 504, 534, 471, 1, 0, 0, 0, 0, 0, 0, 0, 534, 506, 517, 486, 528, 535, 505, 543, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 500, 511, 506, 477, 539, 503, 1, 0, 0, 0, 0, 0, 0, 0, 526, 468, 508, 491, 492, 480, 509, 534, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4179, 4062, 4202, 3999, 4154, 4098, 4015] avg=4096.0000 max=4202 max_rank=3 min=3999 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[545, 531, 501, 503, 538, 502, 505, 471, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 556, 505, 471, 538, 533, 485, 12, 0, 0, 0, 0, 0, 0, 0, 535, 517, 509, 482, 511, 496, 513, 506, 12, 6, 0, 0, 0, 0, 0, 0, 490, 507, 516, 534, 520, 524, 501, 528, 15, 0, 0, 0, 0, 0, 0, 0, 555, 520, 503, 492, 517, 550, 486, 503, 4, 0, 0, 0, 0, 0, 0, 0, 460, 517, 529, 505, 525, 511, 468, 538, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 487, 517, 503, 561, 504, 510, 8, 0, 0, 0, 0, 0, 0, 0, 517, 499, 504, 501, 517, 491, 542, 510, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4087, 4135, 4130, 4053, 4089, 4093] avg=4096.0000 max=4135 max_rank=3 min=4053 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[504, 517, 534, 508, 536, 525, 536, 534, 568, 512, 505, 516, 492, 526, 509, 509, 503, 477, 481, 533, 503, 477, 523, 515, 469, 506, 535, 507, 524, 532, 478, 546, 497, 478, 526, 522, 492, 492, 511, 503, 536, 521, 541, 528, 479, 504, 485, 528, 484, 525, 528, 523, 494, 502, 536, 514, 508, 497, 513, 502, 511, 494, 539, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[516, 529, 530, 532, 484, 542, 483, 508, 555, 507, 515, 538, 500, 514, 506, 502, 510, 482, 538, 505, 491, 518, 526, 504, 541, 530, 536, 514, 541, 506, 484, 478, 532, 519, 520, 518, 551, 499, 510, 512, 500, 460, 513, 523, 474, 503, 511, 521, 496, 495, 549, 494, 473, 500, 501, 497, 524, 550, 506, 531, 502, 512, 479, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[510, 512, 530, 514, 497, 526, 528, 512, 509, 515, 502, 494, 534, 515, 524, 510, 523, 543, 517, 493, 498, 517, 501, 512, 504, 488, 496, 492, 504, 490, 482, 471, 501, 507, 510, 519, 493, 489, 520, 519, 514, 536, 532, 511, 480, 526, 498, 536, 495, 507, 543, 522, 511, 506, 513, 541, 518, 528, 504, 556, 516, 532, 468, 554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[504, 508, 526, 511, 521, 512, 504, 497, 498, 472, 503, 526, 525, 509, 494, 548, 522, 498, 498, 466, 521, 480, 496, 503, 560, 544, 531, 503, 502, 526, 520, 513, 513, 503, 499, 475, 517, 530, 513, 519, 485, 459, 583, 539, 503, 528, 542, 490, 491, 512, 461, 502, 498, 511, 493, 541, 547, 513, 517, 523, 555, 503, 553, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[508, 486, 513, 502, 519, 536, 497, 522, 488, 512, 515, 513, 514, 523, 547, 496, 510, 523, 540, 502, 504, 493, 536, 519, 570, 525, 485, 504, 515, 490, 529, 461, 494, 516, 521, 518, 560, 538, 522, 532, 531, 501, 525, 527, 530, 547, 546, 473, 509, 539, 507, 447, 483, 469, 481, 554, 469, 507, 514, 522, 505, 475, 520, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[517, 503, 507, 504, 535, 514, 485, 510, 493, 505, 525, 542, 516, 503, 482, 496, 523, 489, 501, 499, 512, 497, 494, 535, 546, 519, 537, 505, 522, 520, 496, 506, 514, 519, 518, 525, 518, 508, 559, 552, 562, 550, 483, 511, 536, 518, 483, 476, 550, 490, 517, 522, 509, 495, 503, 495, 497, 493, 484, 532, 478, 517, 525, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[477, 534, 503, 494, 495, 547, 511, 527, 490, 486, 511, 473, 571, 506, 504, 523, 531, 496, 506, 557, 519, 479, 521, 490, 509, 500, 517, 520, 507, 492, 511, 537, 518, 505, 474, 507, 494, 510, 545, 498, 505, 525, 552, 529, 522, 524, 462, 532, 484, 511, 543, 520, 495, 511, 523, 527, 532, 542, 495, 485, 516, 524, 515, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[509, 532, 496, 526, 485, 503, 522, 504, 512, 526, 544, 526, 537, 521, 498, 568, 516, 484, 490, 484, 481, 539, 520, 501, 529, 495, 494, 507, 555, 500, 502, 505, 514, 478, 489, 487, 486, 528, 534, 486, 508, 546, 534, 508, 522, 500, 477, 522, 535, 535, 501, 529, 467, 519, 564, 524, 468, 498, 503, 527, 509, 544, 495, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 517, 534, 508, 536, 515, 536, 534, 0, 0, 0, 0, 0, 0, 0, 0, 568, 512, 505, 516, 475, 526, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 503, 477, 481, 533, 503, 477, 523, 515, 23, 13, 0, 0, 0, 0, 0, 0, 469, 506, 535, 507, 524, 532, 478, 546, 0, 0, 0, 0, 0, 0, 0, 0, 497, 478, 526, 522, 492, 492, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 536, 521, 518, 528, 479, 504, 485, 528, 0, 0, 0, 0, 0, 0, 0, 0, 484, 525, 528, 523, 494, 502, 536, 514, 10, 4, 3, 0, 0, 0, 0, 0, 508, 497, 513, 499, 511, 494, 539, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4120, 4048, 4097, 4021, 4099, 4123, 4076] avg=4096.0000 max=4184 max_rank=0 min=4021 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[510, 512, 530, 514, 497, 516, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 515, 502, 494, 516, 515, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 523, 543, 517, 493, 498, 517, 501, 512, 23, 14, 0, 0, 0, 0, 0, 0, 504, 488, 496, 492, 504, 490, 482, 471, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 510, 519, 493, 489, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 509, 511, 480, 526, 498, 536, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 543, 522, 511, 506, 513, 541, 10, 4, 3, 0, 0, 0, 0, 0, 518, 528, 504, 553, 516, 532, 468, 554, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4085, 4141, 3927, 4058, 4110, 4155, 4173] avg=4096.0000 max=4173 max_rank=7 min=3927 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 529, 530, 532, 484, 532, 483, 508, 0, 0, 0, 0, 0, 0, 0, 0, 555, 507, 515, 538, 483, 514, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 510, 482, 538, 505, 491, 518, 526, 504, 22, 13, 0, 0, 0, 0, 0, 0, 541, 530, 536, 514, 541, 506, 484, 478, 0, 0, 0, 0, 0, 0, 0, 0, 532, 519, 520, 518, 551, 499, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 500, 460, 491, 523, 474, 503, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 549, 494, 473, 500, 501, 497, 10, 4, 3, 0, 0, 0, 0, 0, 524, 550, 506, 528, 502, 512, 479, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4120, 4109, 4130, 4161, 3983, 4022, 4129] avg=4096.0000 max=4161 max_rank=4 min=3983 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 508, 526, 511, 521, 502, 504, 497, 0, 0, 0, 0, 0, 0, 0, 0, 498, 472, 503, 526, 507, 509, 494, 548, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 498, 466, 521, 480, 496, 503, 25, 14, 0, 0, 0, 0, 0, 0, 560, 544, 531, 503, 502, 526, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 499, 475, 517, 530, 513, 519, 0, 0, 0, 0, 0, 0, 0, 0, 485, 459, 558, 539, 503, 528, 542, 490, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 461, 502, 498, 511, 493, 541, 10, 4, 3, 0, 0, 0, 0, 0, 547, 513, 517, 520, 555, 503, 553, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4057, 4023, 4199, 4069, 4104, 4026, 4217] avg=4096.0000 max=4217 max_rank=7 min=4023 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 503, 507, 504, 535, 504, 485, 510, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 525, 542, 498, 503, 482, 496, 0, 0, 0, 0, 0, 0, 0, 0, 523, 489, 501, 499, 512, 497, 494, 535, 21, 14, 0, 0, 0, 0, 0, 0, 546, 519, 537, 505, 522, 520, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 518, 525, 518, 508, 559, 552, 0, 0, 0, 0, 0, 0, 0, 0, 562, 550, 462, 511, 536, 518, 483, 476, 0, 0, 0, 0, 0, 0, 0, 0, 550, 490, 517, 522, 509, 495, 503, 495, 10, 4, 3, 0, 0, 0, 0, 0, 497, 493, 484, 529, 478, 517, 525, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4044, 4085, 4151, 4213, 4098, 4098, 4014] avg=4096.0000 max=4213 max_rank=4 min=4014 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[508, 486, 513, 502, 519, 522, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 488, 512, 515, 513, 489, 523, 547, 496, 0, 0, 0, 0, 0, 0, 0, 0, 510, 523, 540, 502, 504, 493, 536, 519, 27, 18, 0, 0, 0, 0, 0, 0, 570, 525, 485, 504, 515, 490, 529, 461, 5, 0, 0, 0, 0, 0, 0, 0, 494, 516, 521, 518, 560, 538, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 501, 498, 527, 530, 547, 546, 473, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 507, 447, 483, 469, 481, 554, 14, 7, 8, 3, 0, 0, 0, 0, 469, 507, 514, 514, 505, 475, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4083, 4172, 4084, 4193, 4153, 4021, 3993] avg=4096.0000 max=4193 max_rank=4 min=3993 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[477, 534, 503, 494, 495, 536, 511, 527, 0, 0, 0, 0, 0, 0, 0, 0, 490, 486, 511, 473, 552, 506, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 506, 557, 519, 479, 521, 490, 24, 15, 0, 0, 0, 0, 0, 0, 509, 500, 517, 520, 507, 492, 511, 537, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 474, 507, 494, 510, 545, 498, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 528, 529, 522, 524, 462, 532, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 543, 520, 495, 511, 523, 527, 11, 4, 3, 0, 0, 0, 0, 0, 532, 542, 495, 482, 516, 524, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4045, 4138, 4093, 4051, 4127, 4132, 4105] avg=4096.0000 max=4138 max_rank=2 min=4045 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 532, 496, 526, 485, 493, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 526, 544, 526, 519, 521, 498, 568, 0, 0, 0, 0, 0, 0, 0, 0, 516, 484, 490, 484, 481, 539, 520, 501, 23, 14, 0, 0, 0, 0, 0, 0, 529, 495, 494, 507, 555, 500, 502, 505, 0, 0, 0, 0, 0, 0, 0, 0, 514, 478, 489, 487, 486, 528, 534, 486, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 511, 508, 522, 500, 477, 522, 0, 0, 0, 0, 0, 0, 0, 0, 535, 535, 501, 529, 467, 519, 564, 524, 10, 4, 3, 0, 0, 0, 0, 0, 468, 498, 503, 524, 509, 544, 495, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4214, 4052, 4087, 4002, 4094, 4191, 4061] avg=4096.0000 max=4214 max_rank=1 min=4002 min_rank=4 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4096, 4224, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[499, 506, 529, 526, 507, 519, 536, 474, 502, 482, 484, 518, 501, 539, 510, 536, 499, 554, 527, 498, 484, 536, 508, 486, 538, 532, 474, 507, 522, 528, 493, 517, 528, 512, 476, 510, 516, 518, 522, 515, 535, 529, 507, 519, 538, 501, 546, 518, 509, 533, 498, 525, 499, 477, 554, 516, 488, 495, 482, 502, 499, 547, 465, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[516, 521, 490, 515, 500, 510, 515, 513, 517, 492, 507, 534, 511, 504, 497, 510, 508, 476, 518, 470, 469, 487, 500, 523, 528, 536, 533, 547, 517, 508, 518, 518, 525, 522, 498, 488, 533, 479, 524, 500, 549, 529, 550, 540, 475, 490, 508, 515, 533, 508, 488, 504, 514, 522, 492, 486, 533, 506, 514, 494, 530, 525, 541, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[514, 521, 511, 519, 510, 512, 556, 520, 497, 499, 526, 510, 551, 504, 479, 539, 521, 460, 518, 529, 518, 503, 545, 544, 493, 526, 539, 511, 543, 510, 500, 547, 502, 510, 524, 515, 529, 483, 527, 525, 515, 498, 474, 505, 523, 505, 492, 501, 525, 514, 491, 488, 519, 521, 511, 477, 488, 491, 485, 492, 515, 515, 523, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[520, 513, 518, 519, 488, 528, 480, 524, 525, 526, 542, 473, 499, 482, 545, 520, 525, 502, 507, 501, 518, 540, 488, 493, 482, 511, 473, 497, 529, 483, 540, 508, 478, 501, 498, 504, 519, 510, 529, 536, 548, 502, 497, 540, 522, 524, 517, 480, 498, 503, 494, 500, 512, 523, 501, 532, 535, 564, 484, 513, 542, 532, 512, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[537, 503, 501, 493, 494, 532, 510, 489, 531, 505, 542, 497, 543, 549, 540, 532, 505, 507, 508, 526, 500, 520, 495, 478, 499, 508, 510, 529, 476, 510, 464, 519, 473, 520, 486, 520, 530, 502, 521, 543, 502, 522, 538, 504, 488, 514, 498, 502, 516, 476, 512, 507, 492, 553, 517, 525, 481, 515, 498, 529, 524, 539, 534, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[538, 514, 510, 521, 529, 531, 495, 521, 528, 512, 471, 513, 511, 481, 531, 478, 479, 539, 486, 521, 477, 499, 506, 520, 486, 499, 550, 518, 539, 515, 519, 535, 486, 498, 499, 547, 493, 517, 523, 487, 502, 540, 486, 503, 563, 519, 491, 507, 482, 534, 549, 488, 514, 534, 471, 506, 534, 525, 507, 560, 522, 480, 504, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[515, 515, 528, 502, 529, 530, 523, 548, 542, 543, 485, 494, 499, 505, 539, 566, 502, 506, 488, 499, 485, 494, 544, 484, 514, 494, 502, 511, 472, 512, 533, 549, 547, 500, 521, 501, 522, 515, 518, 527, 491, 486, 494, 526, 488, 505, 513, 535, 495, 517, 523, 498, 539, 549, 494, 519, 496, 533, 516, 532, 479, 501, 478, 458] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[505, 499, 516, 526, 492, 514, 557, 509, 497, 495, 498, 500, 522, 553, 524, 511, 516, 519, 450, 470, 550, 476, 499, 503, 528, 491, 530, 478, 537, 480, 545, 453, 508, 495, 542, 514, 487, 519, 523, 519, 503, 524, 494, 537, 504, 500, 507, 499, 520, 497, 577, 511, 561, 535, 494, 528, 530, 507, 516, 515, 513, 523, 493, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[499, 506, 529, 526, 507, 496, 536, 474, 0, 0, 0, 0, 0, 0, 0, 0, 502, 482, 484, 518, 501, 539, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 499, 554, 527, 498, 484, 536, 508, 486, 23, 16, 8, 5, 3, 0, 0, 0, 538, 532, 474, 507, 522, 528, 493, 512, 0, 0, 0, 0, 0, 0, 0, 0, 528, 512, 476, 510, 516, 518, 522, 515, 1, 0, 0, 0, 0, 0, 0, 0, 535, 529, 507, 518, 538, 501, 546, 518, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 498, 525, 499, 474, 554, 516, 0, 0, 0, 0, 0, 0, 0, 0, 488, 495, 482, 502, 499, 539, 465, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4056, 4147, 4106, 4098, 4192, 4108, 3988] avg=4096.0000 max=4192 max_rank=5 min=3988 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[516, 521, 490, 515, 500, 488, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 507, 534, 511, 504, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 508, 476, 518, 470, 469, 487, 500, 523, 22, 15, 8, 5, 3, 0, 0, 0, 528, 536, 533, 547, 517, 508, 518, 513, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 498, 488, 533, 479, 524, 500, 1, 0, 0, 0, 0, 0, 0, 0, 549, 529, 550, 539, 475, 490, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 533, 508, 488, 504, 514, 519, 492, 486, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 514, 494, 530, 517, 541, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4057, 4004, 4200, 4070, 4155, 4044, 4180] avg=4096.0000 max=4200 max_rank=3 min=4004 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16449 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[514, 521, 511, 519, 510, 490, 556, 520, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 526, 510, 551, 504, 479, 523, 0, 0, 0, 0, 0, 0, 0, 0, 521, 460, 518, 529, 518, 503, 545, 544, 22, 16, 8, 5, 3, 0, 0, 0, 493, 526, 539, 511, 543, 510, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 524, 515, 529, 483, 527, 525, 1, 0, 0, 0, 0, 0, 0, 0, 515, 498, 474, 504, 523, 505, 492, 501, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 491, 488, 519, 518, 511, 477, 0, 0, 0, 0, 0, 0, 0, 0, 488, 491, 485, 492, 515, 507, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4089, 4192, 4164, 4116, 4012, 4043, 4011] avg=4096.0000 max=4192 max_rank=2 min=4011 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16182 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 513, 518, 519, 488, 505, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 542, 473, 499, 482, 545, 504, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 507, 501, 518, 540, 488, 493, 23, 16, 8, 5, 3, 0, 0, 0, 482, 511, 473, 497, 529, 483, 540, 503, 0, 0, 0, 0, 0, 0, 0, 0, 478, 501, 498, 504, 519, 510, 529, 536, 1, 0, 0, 0, 0, 0, 0, 0, 548, 502, 497, 539, 522, 524, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 494, 500, 512, 520, 501, 532, 0, 0, 0, 0, 0, 0, 0, 0, 535, 564, 484, 513, 542, 524, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4096, 4129, 4018, 4076, 4129, 4060, 4193] avg=4096.0000 max=4193 max_rank=7 min=4018 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16458 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[537, 503, 501, 493, 494, 509, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 542, 497, 543, 549, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 508, 526, 500, 520, 495, 478, 23, 16, 8, 5, 4, 0, 0, 0, 499, 508, 510, 529, 476, 510, 464, 514, 0, 0, 0, 0, 0, 0, 0, 0, 473, 520, 486, 520, 530, 502, 521, 543, 1, 0, 0, 0, 0, 0, 0, 0, 502, 522, 538, 503, 488, 514, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 516, 476, 512, 507, 492, 549, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 481, 515, 498, 529, 524, 531, 534, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4223, 4095, 4010, 4096, 4067, 4094, 4147] avg=4096.0000 max=4223 max_rank=1 min=4010 min_rank=3 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16404 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[538, 514, 510, 521, 529, 508, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 528, 512, 471, 513, 511, 481, 531, 464, 0, 0, 0, 0, 0, 0, 0, 0, 479, 539, 486, 521, 477, 499, 506, 520, 23, 14, 7, 5, 4, 0, 0, 0, 486, 499, 550, 518, 539, 515, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 486, 498, 499, 547, 493, 517, 523, 487, 1, 0, 0, 0, 0, 0, 0, 0, 502, 540, 486, 502, 563, 519, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 482, 534, 549, 488, 514, 530, 471, 506, 0, 0, 0, 0, 0, 0, 0, 0, 534, 525, 507, 560, 522, 473, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4011, 4080, 4156, 4051, 4110, 4074, 4150] avg=4096.0000 max=4156 max_rank=3 min=4011 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 515, 528, 502, 529, 502, 523, 548, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 485, 494, 499, 505, 539, 544, 0, 0, 0, 0, 0, 0, 0, 0, 502, 506, 488, 499, 485, 494, 544, 484, 28, 22, 13, 11, 8, 0, 0, 0, 514, 494, 502, 511, 472, 512, 533, 538, 0, 0, 0, 0, 0, 0, 0, 0, 547, 500, 521, 501, 522, 515, 518, 527, 2, 0, 0, 0, 0, 0, 0, 0, 491, 486, 494, 524, 488, 505, 513, 535, 0, 0, 0, 0, 0, 0, 0, 0, 495, 517, 523, 498, 539, 541, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 496, 533, 516, 532, 479, 488, 478, 458, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4151, 4084, 4076, 4153, 4036, 4126, 3980] avg=4096.0000 max=4162 max_rank=0 min=3980 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16295 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 499, 516, 526, 492, 491, 557, 509, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 498, 500, 522, 553, 524, 496, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 450, 470, 550, 476, 499, 503, 23, 15, 8, 4, 4, 0, 0, 0, 528, 491, 530, 478, 537, 480, 545, 449, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 542, 514, 487, 519, 523, 519, 1, 0, 0, 0, 0, 0, 0, 0, 503, 524, 494, 536, 504, 500, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 577, 511, 561, 531, 494, 528, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 516, 515, 513, 515, 493, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 4037, 4038, 4108, 4067, 4219, 4119] avg=4096.0000 max=4219 max_rank=6 min=4037 min_rank=2 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16513 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 256, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 15:43:12] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 277.3 | throughput per GPU (TFLOP/s/GPU): 445.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.867383E+00 | loss scale: 1.0 | grad norm: 19.127 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[528, 491, 491, 528, 512, 501, 540, 541, 526, 544, 524, 504, 550, 519, 510, 482, 526, 496, 507, 484, 514, 459, 515, 511, 493, 519, 481, 529, 518, 536, 524, 505, 510, 488, 482, 524, 513, 515, 500, 499, 513, 523, 512, 513, 491, 509, 509, 516, 508, 518, 509, 511, 496, 567, 518, 532, 506, 499, 495, 509, 509, 517, 514, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[547, 493, 515, 494, 519, 517, 515, 529, 534, 535, 528, 527, 494, 496, 504, 506, 505, 496, 517, 500, 521, 471, 509, 522, 497, 539, 513, 511, 480, 511, 540, 532, 489, 507, 551, 542, 520, 514, 532, 542, 532, 478, 495, 488, 524, 520, 517, 518, 463, 484, 478, 542, 528, 498, 507, 522, 493, 505, 497, 518, 526, 503, 512, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[497, 525, 504, 508, 526, 496, 507, 518, 558, 505, 504, 511, 537, 488, 522, 522, 493, 495, 491, 515, 509, 477, 486, 494, 546, 531, 519, 506, 525, 454, 555, 533, 502, 512, 545, 464, 535, 510, 507, 495, 500, 514, 513, 521, 516, 528, 525, 468, 504, 524, 554, 479, 537, 501, 514, 516, 495, 537, 531, 525, 509, 531, 473, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[489, 551, 520, 490, 518, 512, 541, 514, 496, 482, 522, 483, 510, 491, 484, 496, 517, 519, 537, 496, 487, 541, 508, 539, 545, 543, 491, 546, 479, 513, 507, 527, 531, 534, 472, 518, 508, 533, 543, 514, 510, 512, 524, 519, 496, 534, 517, 497, 508, 505, 540, 472, 484, 523, 518, 521, 500, 518, 517, 501, 503, 489, 491, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[513, 524, 546, 494, 503, 545, 526, 476, 463, 509, 493, 520, 490, 502, 518, 533, 524, 524, 505, 531, 497, 516, 484, 502, 527, 522, 517, 494, 505, 528, 544, 503, 559, 501, 520, 489, 538, 502, 465, 486, 510, 485, 504, 510, 524, 522, 483, 532, 548, 549, 498, 507, 512, 505, 503, 521, 524, 526, 484, 505, 504, 534, 505, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[519, 506, 484, 529, 501, 457, 522, 540, 521, 508, 530, 463, 497, 560, 512, 541, 475, 510, 484, 486, 531, 511, 527, 515, 538, 550, 518, 513, 506, 518, 484, 512, 507, 543, 471, 509, 527, 511, 526, 492, 538, 573, 496, 501, 500, 522, 530, 495, 491, 491, 552, 491, 495, 536, 481, 527, 519, 532, 518, 495, 519, 474, 492, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[501, 487, 528, 517, 517, 483, 504, 510, 525, 555, 503, 513, 492, 529, 516, 493, 505, 525, 478, 513, 514, 533, 493, 530, 532, 517, 522, 505, 498, 524, 553, 546, 527, 519, 499, 485, 523, 515, 511, 518, 515, 484, 490, 508, 514, 470, 524, 524, 490, 544, 501, 525, 498, 461, 519, 490, 486, 559, 550, 503, 539, 500, 525, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[505, 476, 463, 511, 504, 450, 561, 537, 510, 507, 521, 519, 544, 509, 507, 521, 545, 515, 531, 518, 491, 497, 521, 509, 494, 550, 511, 530, 516, 514, 505, 546, 503, 535, 528, 499, 472, 492, 533, 494, 515, 506, 510, 509, 512, 542, 485, 521, 526, 481, 503, 491, 532, 523, 512, 501, 522, 521, 493, 521, 491, 556, 464, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[528, 491, 491, 528, 512, 501, 540, 541, 6, 0, 0, 0, 0, 0, 0, 0, 526, 538, 524, 504, 550, 519, 510, 482, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 507, 484, 514, 459, 515, 511, 32, 0, 0, 0, 0, 0, 0, 0, 493, 470, 481, 529, 518, 536, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 482, 524, 513, 515, 500, 499, 1, 0, 0, 0, 0, 0, 0, 0, 513, 523, 512, 513, 491, 509, 509, 516, 7, 0, 0, 0, 0, 0, 0, 0, 508, 518, 509, 511, 496, 567, 518, 532, 10, 0, 0, 0, 0, 0, 0, 0, 506, 499, 495, 509, 509, 517, 514, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4153, 4044, 4056, 4032, 4093, 4169, 4083] avg=4096.0000 max=4169 max_rank=6 min=4032 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[519, 506, 484, 529, 501, 457, 522, 540, 5, 0, 0, 0, 0, 0, 0, 0, 521, 503, 530, 463, 497, 560, 512, 541, 0, 0, 0, 0, 0, 0, 0, 0, 475, 510, 484, 486, 531, 511, 527, 515, 34, 0, 0, 0, 0, 0, 0, 0, 538, 499, 518, 513, 506, 518, 484, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 543, 471, 509, 527, 511, 526, 492, 1, 0, 0, 0, 0, 0, 0, 0, 538, 573, 496, 501, 500, 522, 530, 495, 7, 0, 0, 0, 0, 0, 0, 0, 491, 491, 552, 491, 495, 536, 481, 527, 10, 0, 0, 0, 0, 0, 0, 0, 519, 532, 518, 495, 519, 474, 492, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4127, 4073, 4088, 4087, 4162, 4074, 4094] avg=4096.0000 max=4162 max_rank=5 min=4063 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[547, 493, 515, 494, 519, 517, 515, 529, 5, 0, 0, 0, 0, 0, 0, 0, 534, 530, 528, 527, 494, 496, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 505, 496, 517, 500, 521, 471, 509, 522, 34, 0, 0, 0, 0, 0, 0, 0, 497, 488, 513, 511, 480, 511, 540, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 507, 551, 542, 520, 514, 532, 542, 1, 0, 0, 0, 0, 0, 0, 0, 532, 478, 495, 488, 524, 520, 517, 518, 7, 0, 0, 0, 0, 0, 0, 0, 463, 484, 478, 542, 528, 498, 507, 522, 10, 0, 0, 0, 0, 0, 0, 0, 493, 505, 497, 518, 526, 503, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4119, 4075, 4072, 4198, 4079, 4032, 4059] avg=4096.0000 max=4198 max_rank=4 min=4032 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[497, 525, 504, 508, 526, 496, 507, 518, 5, 0, 0, 0, 0, 0, 0, 0, 558, 500, 504, 511, 537, 488, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 493, 495, 491, 515, 509, 477, 486, 494, 33, 0, 0, 0, 0, 0, 0, 0, 546, 481, 519, 506, 525, 454, 555, 533, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 545, 464, 535, 510, 507, 495, 1, 0, 0, 0, 0, 0, 0, 0, 500, 514, 513, 521, 516, 528, 525, 468, 7, 0, 0, 0, 0, 0, 0, 0, 504, 524, 554, 479, 537, 501, 514, 516, 10, 0, 0, 0, 0, 0, 0, 0, 495, 537, 531, 525, 509, 531, 473, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4142, 3993, 4119, 4071, 4092, 4139, 4126] avg=4096.0000 max=4142 max_rank=1 min=3993 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[489, 551, 520, 490, 518, 512, 541, 514, 5, 0, 0, 0, 0, 0, 0, 0, 496, 477, 522, 483, 510, 491, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 537, 496, 487, 541, 508, 539, 34, 0, 0, 0, 0, 0, 0, 0, 545, 492, 491, 546, 479, 513, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 531, 534, 472, 518, 508, 533, 543, 514, 1, 0, 0, 0, 0, 0, 0, 0, 510, 512, 524, 519, 496, 534, 517, 497, 7, 0, 0, 0, 0, 0, 0, 0, 508, 505, 540, 472, 484, 523, 518, 521, 10, 0, 0, 0, 0, 0, 0, 0, 500, 518, 517, 501, 503, 489, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 3959, 4178, 4100, 4154, 4116, 4081, 4040] avg=4096.0000 max=4178 max_rank=2 min=3959 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[513, 524, 546, 494, 503, 545, 526, 476, 5, 0, 0, 0, 0, 0, 0, 0, 463, 504, 493, 520, 490, 502, 518, 533, 0, 0, 0, 0, 0, 0, 0, 0, 524, 524, 505, 531, 497, 516, 484, 502, 33, 0, 0, 0, 0, 0, 0, 0, 527, 472, 517, 494, 505, 528, 544, 503, 0, 0, 0, 0, 0, 0, 0, 0, 559, 501, 520, 489, 538, 502, 465, 486, 1, 0, 0, 0, 0, 0, 0, 0, 510, 485, 504, 510, 524, 522, 483, 532, 7, 0, 0, 0, 0, 0, 0, 0, 548, 549, 498, 507, 512, 505, 503, 521, 10, 0, 0, 0, 0, 0, 0, 0, 524, 526, 484, 505, 504, 534, 505, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4023, 4116, 4090, 4061, 4077, 4153, 4116] avg=4096.0000 max=4153 max_rank=6 min=4023 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 487, 528, 517, 517, 483, 504, 510, 6, 0, 0, 0, 0, 0, 0, 0, 525, 549, 503, 513, 492, 529, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 478, 513, 514, 533, 493, 530, 32, 0, 0, 0, 0, 0, 0, 0, 532, 468, 522, 505, 498, 524, 553, 546, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 499, 485, 523, 515, 511, 518, 1, 0, 0, 0, 0, 0, 0, 0, 515, 484, 490, 508, 514, 470, 524, 524, 7, 0, 0, 0, 0, 0, 0, 0, 490, 544, 501, 525, 498, 461, 519, 490, 10, 0, 0, 0, 0, 0, 0, 0, 486, 559, 550, 503, 539, 500, 525, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4120, 4123, 4148, 4098, 4036, 4038, 4152] avg=4096.0000 max=4152 max_rank=7 min=4036 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 476, 463, 511, 504, 450, 561, 537, 9, 6, 0, 0, 0, 0, 0, 0, 510, 494, 521, 519, 544, 509, 507, 521, 0, 0, 0, 0, 0, 0, 0, 0, 545, 515, 531, 518, 491, 497, 521, 509, 39, 0, 0, 0, 0, 0, 0, 0, 494, 479, 511, 530, 516, 514, 505, 546, 0, 0, 0, 0, 0, 0, 0, 0, 503, 535, 528, 499, 472, 492, 533, 494, 7, 4, 0, 0, 0, 0, 0, 0, 515, 506, 510, 509, 512, 542, 485, 521, 13, 0, 0, 0, 0, 0, 0, 0, 526, 481, 503, 491, 532, 523, 512, 501, 13, 0, 0, 0, 0, 0, 0, 0, 522, 521, 493, 521, 491, 556, 464, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4125, 4166, 4095, 4067, 4113, 4082, 4098] avg=4096.0000 max=4166 max_rank=2 min=4022 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[464, 499, 497, 515, 505, 516, 522, 496, 522, 488, 500, 521, 502, 505, 524, 555, 522, 533, 494, 489, 545, 517, 499, 499, 529, 544, 517, 517, 500, 529, 502, 513, 519, 538, 532, 517, 483, 531, 490, 478, 497, 527, 529, 545, 505, 552, 516, 554, 518, 510, 493, 517, 506, 498, 501, 493, 509, 503, 503, 522, 493, 498, 514, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[517, 515, 496, 509, 529, 550, 527, 507, 479, 545, 489, 519, 509, 525, 500, 501, 510, 530, 506, 510, 481, 508, 521, 500, 546, 505, 517, 535, 505, 526, 518, 528, 524, 527, 524, 510, 479, 491, 512, 513, 489, 518, 494, 516, 468, 513, 536, 517, 520, 549, 520, 496, 513, 515, 471, 547, 508, 501, 527, 523, 483, 501, 499, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[546, 472, 508, 542, 528, 502, 513, 505, 499, 503, 502, 527, 534, 539, 519, 516, 547, 509, 508, 516, 495, 498, 524, 538, 510, 525, 472, 527, 497, 508, 504, 478, 510, 541, 533, 491, 485, 525, 495, 497, 520, 512, 521, 497, 513, 531, 528, 500, 495, 487, 538, 536, 507, 503, 516, 478, 511, 498, 538, 501, 490, 542, 519, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[507, 562, 519, 555, 538, 494, 513, 526, 516, 535, 494, 485, 521, 462, 527, 516, 510, 504, 505, 504, 515, 525, 548, 536, 522, 497, 517, 493, 489, 537, 503, 532, 550, 535, 510, 490, 554, 520, 502, 511, 503, 534, 529, 523, 512, 496, 490, 516, 506, 528, 494, 495, 517, 483, 483, 487, 491, 488, 498, 522, 472, 478, 512, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[510, 510, 520, 501, 514, 497, 501, 489, 490, 478, 527, 483, 544, 515, 511, 482, 545, 494, 533, 539, 502, 543, 530, 503, 514, 535, 517, 511, 533, 520, 486, 531, 500, 475, 527, 501, 507, 507, 526, 487, 491, 533, 510, 515, 562, 527, 536, 497, 482, 520, 521, 524, 492, 531, 516, 511, 505, 530, 506, 536, 489, 530, 508, 458] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[526, 511, 547, 530, 485, 475, 496, 540, 486, 520, 551, 526, 524, 541, 497, 502, 533, 506, 495, 510, 516, 534, 523, 507, 543, 506, 497, 504, 499, 494, 528, 499, 495, 509, 512, 473, 474, 520, 531, 522, 540, 502, 543, 509, 476, 513, 560, 539, 515, 492, 512, 512, 475, 543, 504, 499, 549, 497, 499, 503, 498, 501, 509, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[524, 493, 513, 498, 533, 483, 499, 501, 531, 518, 500, 521, 509, 499, 492, 470, 514, 518, 526, 509, 512, 518, 511, 499, 498, 495, 504, 529, 530, 494, 531, 511, 488, 559, 496, 527, 542, 524, 530, 511, 508, 535, 532, 505, 515, 507, 519, 490, 520, 533, 503, 522, 515, 468, 484, 528, 516, 539, 516, 506, 504, 517, 525, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[526, 507, 520, 514, 526, 493, 562, 537, 508, 501, 497, 496, 500, 519, 501, 464, 514, 470, 529, 505, 545, 466, 526, 509, 514, 523, 533, 543, 533, 571, 502, 511, 545, 508, 503, 500, 525, 501, 479, 526, 505, 479, 496, 524, 515, 494, 489, 568, 486, 508, 482, 523, 477, 523, 531, 481, 521, 551, 493, 509, 481, 521, 503, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[464, 499, 497, 503, 505, 516, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 522, 488, 500, 521, 502, 505, 524, 555, 12, 10, 0, 0, 0, 0, 0, 0, 503, 533, 494, 489, 545, 517, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 529, 544, 517, 517, 500, 503, 502, 513, 0, 0, 0, 0, 0, 0, 0, 0, 519, 538, 532, 517, 483, 531, 490, 478, 2, 0, 0, 0, 0, 0, 0, 0, 497, 527, 529, 545, 505, 552, 516, 519, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 493, 517, 506, 498, 501, 493, 35, 6, 0, 0, 0, 0, 0, 0, 509, 503, 503, 522, 493, 498, 514, 497, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4139, 4079, 4125, 4090, 4190, 4077, 4066] avg=4096.0000 max=4190 max_rank=5 min=4002 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 515, 496, 497, 529, 550, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 479, 545, 489, 519, 509, 525, 500, 501, 12, 10, 0, 0, 0, 0, 0, 0, 491, 530, 506, 510, 481, 508, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 517, 535, 505, 500, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 524, 510, 479, 491, 512, 513, 2, 0, 0, 0, 0, 0, 0, 0, 489, 518, 494, 516, 468, 513, 536, 484, 0, 0, 0, 0, 0, 0, 0, 0, 520, 549, 520, 496, 513, 515, 471, 547, 33, 6, 0, 0, 0, 0, 0, 0, 508, 501, 527, 523, 483, 501, 499, 501, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4089, 4047, 4154, 4082, 4018, 4170, 4070] avg=4096.0000 max=4170 max_rank=6 min=4018 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[507, 562, 519, 542, 538, 494, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 516, 535, 494, 485, 521, 462, 527, 516, 12, 11, 0, 0, 0, 0, 0, 0, 491, 504, 505, 504, 515, 525, 548, 536, 0, 0, 0, 0, 0, 0, 0, 0, 522, 497, 517, 493, 489, 510, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 550, 535, 510, 490, 554, 520, 502, 511, 2, 0, 0, 0, 0, 0, 0, 0, 503, 534, 529, 523, 512, 496, 490, 484, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 494, 495, 517, 483, 483, 487, 32, 6, 0, 0, 0, 0, 0, 0, 491, 488, 498, 522, 472, 478, 512, 532, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 4079, 4128, 4063, 4174, 4071, 4031, 4021] avg=4096.0000 max=4201 max_rank=0 min=4021 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[546, 472, 508, 529, 528, 502, 513, 505, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 502, 527, 534, 539, 519, 516, 13, 11, 0, 0, 0, 0, 0, 0, 526, 509, 508, 516, 495, 498, 524, 538, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 472, 527, 497, 483, 504, 478, 0, 0, 0, 0, 0, 0, 0, 0, 510, 541, 533, 491, 485, 525, 495, 497, 2, 0, 0, 0, 0, 0, 0, 0, 520, 512, 521, 497, 513, 531, 528, 469, 0, 0, 0, 0, 0, 0, 0, 0, 495, 487, 538, 536, 507, 503, 516, 478, 31, 5, 0, 0, 0, 0, 0, 0, 511, 498, 538, 501, 490, 542, 519, 499, 20, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4163, 4114, 3996, 4079, 4091, 4096, 4126] avg=4096.0000 max=4163 max_rank=1 min=3996 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 510, 520, 489, 514, 497, 501, 489, 0, 0, 0, 0, 0, 0, 0, 0, 490, 478, 527, 483, 544, 515, 511, 482, 12, 10, 0, 0, 0, 0, 0, 0, 525, 494, 533, 539, 502, 543, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 514, 535, 517, 511, 533, 495, 486, 531, 0, 0, 0, 0, 0, 0, 0, 0, 500, 475, 527, 501, 507, 507, 526, 487, 2, 0, 0, 0, 0, 0, 0, 0, 491, 533, 510, 515, 562, 527, 536, 466, 0, 0, 0, 0, 0, 0, 0, 0, 482, 520, 521, 524, 492, 531, 516, 511, 31, 5, 0, 0, 0, 0, 0, 0, 505, 530, 506, 536, 489, 530, 508, 458, 20, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4052, 4169, 4122, 4032, 4140, 4133, 4090] avg=4096.0000 max=4169 max_rank=2 min=4030 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[524, 493, 513, 486, 533, 483, 499, 501, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 500, 521, 509, 499, 492, 470, 12, 10, 0, 0, 0, 0, 0, 0, 495, 518, 526, 509, 512, 518, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 504, 529, 530, 470, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 488, 559, 496, 527, 542, 524, 530, 511, 2, 0, 0, 0, 0, 0, 0, 0, 508, 535, 532, 505, 515, 507, 519, 459, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 503, 522, 515, 468, 484, 528, 31, 5, 0, 0, 0, 0, 0, 0, 516, 539, 516, 506, 504, 517, 525, 501, 19, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4062, 4088, 4068, 4179, 4080, 4109, 4150] avg=4096.0000 max=4179 max_rank=4 min=4032 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 511, 547, 509, 485, 475, 496, 540, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 551, 526, 524, 541, 497, 502, 15, 14, 0, 0, 0, 0, 0, 0, 507, 506, 495, 510, 516, 534, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 543, 506, 497, 504, 499, 462, 528, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 512, 473, 474, 520, 531, 522, 7, 0, 0, 0, 0, 0, 0, 0, 540, 502, 543, 509, 476, 513, 560, 501, 0, 0, 0, 0, 0, 0, 0, 0, 515, 492, 512, 512, 475, 543, 504, 499, 38, 9, 0, 0, 0, 0, 0, 0, 549, 497, 499, 503, 498, 501, 509, 491, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4176, 4098, 4038, 4043, 4144, 4099, 4081] avg=4096.0000 max=4176 max_rank=1 min=4038 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 507, 520, 502, 526, 493, 562, 537, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 497, 496, 500, 519, 501, 464, 12, 10, 0, 0, 0, 0, 0, 0, 495, 470, 529, 505, 545, 466, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 533, 543, 533, 543, 502, 511, 0, 0, 0, 0, 0, 0, 0, 0, 545, 508, 503, 500, 525, 501, 479, 526, 2, 0, 0, 0, 0, 0, 0, 0, 505, 479, 496, 524, 515, 494, 489, 532, 0, 0, 0, 0, 0, 0, 0, 0, 486, 508, 482, 523, 477, 523, 531, 481, 36, 6, 0, 0, 0, 0, 0, 0, 521, 551, 493, 509, 481, 521, 503, 556, 22, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4008, 4045, 4202, 4089, 4034, 4053, 4164] avg=4096.0000 max=4202 max_rank=3 min=4008 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[498, 512, 574, 505, 503, 497, 494, 528, 524, 520, 494, 531, 514, 515, 503, 522, 506, 494, 552, 493, 485, 494, 496, 492, 531, 534, 513, 512, 484, 520, 521, 524, 530, 513, 513, 507, 545, 502, 519, 483, 463, 547, 539, 519, 500, 527, 526, 535, 519, 508, 491, 533, 464, 462, 512, 486, 523, 496, 523, 529, 512, 525, 499, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[547, 525, 511, 488, 552, 502, 458, 489, 505, 527, 503, 515, 487, 480, 491, 518, 522, 517, 539, 512, 515, 502, 499, 492, 521, 520, 535, 529, 496, 521, 476, 491, 501, 526, 536, 517, 541, 525, 538, 519, 493, 538, 504, 521, 527, 495, 520, 526, 517, 536, 512, 479, 511, 516, 505, 499, 494, 508, 525, 487, 524, 521, 518, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[470, 531, 516, 501, 498, 520, 509, 529, 540, 487, 508, 533, 512, 522, 516, 503, 508, 493, 518, 534, 551, 533, 513, 486, 541, 511, 510, 501, 536, 495, 494, 520, 538, 494, 437, 504, 482, 506, 531, 519, 523, 508, 508, 491, 499, 510, 505, 519, 530, 504, 549, 522, 518, 519, 502, 504, 480, 547, 482, 531, 521, 516, 537, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[491, 527, 543, 493, 529, 504, 527, 498, 515, 559, 540, 502, 503, 496, 507, 513, 561, 513, 486, 516, 480, 521, 507, 516, 517, 505, 501, 544, 484, 492, 515, 543, 474, 523, 528, 491, 513, 528, 526, 538, 527, 478, 520, 528, 506, 547, 501, 504, 487, 515, 503, 514, 512, 488, 541, 523, 453, 501, 510, 507, 510, 514, 484, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[510, 527, 501, 506, 539, 490, 543, 483, 545, 500, 512, 517, 497, 525, 543, 523, 481, 480, 541, 527, 500, 496, 515, 536, 526, 507, 510, 488, 549, 535, 484, 509, 520, 533, 528, 511, 471, 484, 547, 461, 486, 519, 519, 510, 503, 508, 521, 537, 521, 536, 529, 512, 522, 535, 494, 502, 499, 501, 482, 530, 483, 524, 519, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[552, 534, 489, 501, 517, 515, 488, 517, 529, 551, 484, 496, 483, 506, 532, 509, 514, 506, 514, 538, 505, 503, 495, 486, 490, 524, 557, 548, 524, 519, 501, 538, 519, 545, 505, 495, 476, 467, 527, 497, 538, 478, 545, 503, 514, 500, 521, 518, 540, 529, 524, 547, 517, 490, 522, 496, 474, 545, 499, 492, 505, 506, 462, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[487, 513, 517, 540, 504, 527, 500, 541, 528, 513, 563, 517, 493, 476, 500, 525, 520, 512, 518, 508, 467, 518, 515, 523, 503, 505, 537, 531, 522, 526, 472, 523, 484, 523, 558, 519, 450, 550, 541, 516, 503, 503, 494, 515, 518, 516, 508, 471, 542, 510, 519, 540, 526, 515, 509, 500, 504, 508, 530, 509, 420, 512, 517, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[513, 546, 538, 524, 477, 518, 542, 513, 457, 481, 515, 493, 485, 494, 519, 527, 511, 523, 528, 479, 507, 523, 517, 526, 500, 473, 554, 537, 520, 520, 493, 501, 500, 539, 501, 505, 492, 504, 506, 502, 514, 520, 535, 485, 555, 496, 522, 488, 491, 528, 515, 534, 515, 499, 486, 537, 529, 496, 508, 504, 535, 522, 534, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[498, 500, 574, 505, 503, 497, 494, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 494, 531, 514, 515, 503, 522, 3, 0, 0, 0, 0, 0, 0, 0, 506, 494, 552, 493, 485, 494, 496, 492, 9, 1, 0, 0, 0, 0, 0, 0, 531, 534, 490, 512, 484, 520, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 530, 513, 513, 507, 545, 502, 519, 483, 3, 2, 0, 0, 0, 0, 0, 0, 463, 547, 533, 519, 500, 527, 526, 535, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 491, 522, 464, 462, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0, 523, 496, 523, 529, 512, 525, 499, 533, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4126, 4022, 4116, 4117, 4150, 3964, 4174] avg=4096.0000 max=4174 max_rank=7 min=3964 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[470, 517, 516, 501, 498, 520, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 540, 487, 508, 533, 512, 522, 516, 503, 3, 0, 0, 0, 0, 0, 0, 0, 508, 493, 518, 534, 551, 533, 513, 486, 9, 2, 0, 0, 0, 0, 0, 0, 541, 511, 487, 501, 536, 495, 494, 520, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 437, 504, 482, 506, 531, 519, 2, 2, 0, 0, 0, 0, 0, 0, 523, 508, 503, 491, 499, 510, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 549, 511, 518, 519, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 480, 547, 482, 531, 521, 516, 537, 493, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4124, 4147, 4085, 4015, 4058, 4137, 4142] avg=4096.0000 max=4147 max_rank=2 min=4015 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[552, 520, 489, 501, 517, 515, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 529, 551, 484, 496, 483, 506, 532, 509, 3, 0, 0, 0, 0, 0, 0, 0, 514, 506, 514, 538, 505, 503, 495, 486, 9, 2, 0, 0, 0, 0, 0, 0, 490, 524, 532, 548, 524, 519, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 519, 545, 505, 495, 476, 467, 527, 497, 3, 2, 0, 0, 0, 0, 0, 0, 538, 478, 539, 503, 514, 500, 521, 518, 0, 0, 0, 0, 0, 0, 0, 0, 540, 529, 524, 536, 517, 490, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 474, 545, 499, 492, 505, 506, 462, 507, 25, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4093, 4072, 4176, 4036, 4111, 4154, 4027] avg=4096.0000 max=4176 max_rank=3 min=4027 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[547, 512, 511, 488, 552, 502, 458, 489, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 503, 515, 487, 480, 491, 518, 3, 0, 0, 0, 0, 0, 0, 0, 522, 517, 539, 512, 515, 502, 499, 492, 8, 1, 0, 0, 0, 0, 0, 0, 521, 520, 511, 529, 496, 521, 476, 491, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 536, 517, 541, 525, 538, 519, 2, 2, 0, 0, 0, 0, 0, 0, 493, 538, 499, 521, 527, 495, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 517, 536, 512, 469, 511, 516, 505, 499, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 525, 487, 524, 521, 518, 504, 24, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4029, 4107, 4065, 4207, 4119, 4065, 4117] avg=4096.0000 max=4207 max_rank=4 min=4029 min_rank=1 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[510, 513, 501, 506, 539, 490, 543, 483, 0, 0, 0, 0, 0, 0, 0, 0, 545, 500, 512, 517, 497, 525, 543, 523, 3, 0, 0, 0, 0, 0, 0, 0, 481, 480, 541, 527, 500, 496, 515, 536, 9, 2, 0, 0, 0, 0, 0, 0, 526, 507, 487, 488, 549, 535, 484, 509, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 528, 511, 471, 484, 547, 461, 2, 2, 0, 0, 0, 0, 0, 0, 486, 519, 514, 510, 503, 508, 521, 537, 0, 0, 0, 0, 0, 0, 0, 0, 521, 536, 529, 501, 522, 535, 494, 502, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 482, 530, 483, 524, 519, 476, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4165, 4087, 4085, 4059, 4098, 4140, 4049] avg=4096.0000 max=4165 max_rank=1 min=4049 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[487, 501, 517, 540, 504, 527, 500, 541, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 563, 517, 493, 476, 500, 525, 2, 0, 0, 0, 0, 0, 0, 0, 520, 512, 518, 508, 467, 518, 515, 523, 9, 1, 0, 0, 0, 0, 0, 0, 503, 505, 513, 531, 522, 526, 472, 523, 0, 0, 0, 0, 0, 0, 0, 0, 484, 523, 558, 519, 450, 550, 541, 516, 2, 2, 0, 0, 0, 0, 0, 0, 503, 503, 490, 515, 518, 516, 508, 471, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 519, 529, 526, 515, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 530, 509, 420, 512, 517, 494, 24, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4117, 4091, 4095, 4145, 4024, 4150, 4029] avg=4096.0000 max=4150 max_rank=6 min=4024 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[491, 507, 543, 493, 529, 504, 527, 498, 0, 0, 0, 0, 0, 0, 0, 0, 515, 559, 540, 502, 503, 496, 507, 513, 5, 0, 0, 0, 0, 0, 0, 0, 561, 513, 486, 516, 480, 521, 507, 516, 12, 5, 0, 0, 0, 0, 0, 0, 517, 505, 473, 544, 484, 492, 515, 543, 0, 0, 0, 0, 0, 0, 0, 0, 474, 523, 528, 491, 513, 528, 526, 538, 7, 7, 0, 0, 0, 0, 0, 0, 527, 478, 508, 528, 506, 547, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 487, 515, 503, 495, 512, 488, 541, 523, 0, 0, 0, 0, 0, 0, 0, 0, 453, 501, 510, 507, 510, 514, 484, 526, 28, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4140, 4117, 4073, 4135, 4099, 4064, 4048] avg=4096.0000 max=4140 max_rank=1 min=4048 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 532, 538, 524, 477, 518, 542, 513, 0, 0, 0, 0, 0, 0, 0, 0, 457, 481, 515, 493, 485, 494, 519, 527, 3, 0, 0, 0, 0, 0, 0, 0, 511, 523, 528, 479, 507, 523, 517, 526, 9, 2, 0, 0, 0, 0, 0, 0, 500, 473, 529, 537, 520, 520, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 500, 539, 501, 505, 492, 504, 506, 502, 3, 2, 0, 0, 0, 0, 0, 0, 514, 520, 529, 485, 555, 496, 522, 488, 0, 0, 0, 0, 0, 0, 0, 0, 491, 528, 515, 523, 515, 499, 486, 537, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 508, 504, 535, 522, 534, 517, 25, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 3974, 4125, 4073, 4054, 4109, 4094, 4182] avg=4096.0000 max=4182 max_rank=7 min=3974 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[485, 547, 516, 556, 517, 523, 536, 532, 489, 536, 499, 491, 558, 494, 480, 527, 525, 532, 488, 478, 547, 497, 513, 520, 503, 506, 514, 544, 522, 513, 484, 535, 488, 511, 530, 496, 491, 528, 476, 492, 503, 541, 520, 522, 540, 510, 505, 528, 495, 512, 526, 505, 505, 513, 464, 511, 480, 507, 541, 467, 500, 526, 511, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[524, 498, 509, 554, 487, 506, 492, 544, 511, 514, 503, 494, 494, 488, 481, 498, 492, 528, 540, 513, 478, 526, 486, 493, 546, 480, 521, 561, 467, 513, 537, 532, 519, 497, 502, 522, 520, 525, 517, 533, 554, 499, 517, 532, 540, 501, 551, 493, 516, 479, 532, 497, 515, 526, 500, 500, 526, 536, 520, 518, 518, 498, 465, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[501, 535, 533, 523, 554, 550, 519, 499, 501, 506, 520, 545, 521, 522, 494, 518, 538, 507, 522, 489, 493, 546, 504, 523, 519, 530, 507, 483, 506, 514, 500, 502, 506, 473, 512, 531, 512, 534, 528, 517, 503, 473, 479, 456, 486, 517, 511, 499, 533, 536, 485, 478, 518, 535, 497, 498, 507, 512, 534, 525, 486, 500, 549, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[506, 518, 508, 500, 530, 485, 511, 522, 488, 507, 515, 527, 499, 532, 506, 518, 513, 516, 500, 490, 507, 562, 491, 532, 520, 533, 525, 507, 520, 500, 488, 519, 507, 524, 535, 529, 493, 541, 500, 542, 519, 529, 510, 490, 529, 540, 498, 495, 522, 507, 456, 546, 522, 481, 500, 514, 526, 494, 479, 503, 511, 521, 504, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[541, 496, 456, 502, 519, 544, 503, 548, 516, 516, 491, 545, 514, 496, 512, 503, 545, 527, 491, 505, 487, 514, 517, 461, 526, 514, 523, 567, 529, 466, 534, 508, 499, 525, 488, 493, 511, 493, 573, 535, 498, 515, 507, 516, 486, 514, 494, 533, 496, 512, 518, 524, 537, 506, 493, 515, 497, 495, 527, 520, 506, 518, 481, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[514, 523, 471, 522, 487, 524, 522, 491, 475, 504, 519, 510, 504, 496, 540, 496, 499, 513, 500, 516, 478, 478, 532, 491, 516, 544, 509, 520, 492, 509, 525, 519, 503, 520, 500, 537, 566, 503, 522, 511, 509, 513, 516, 466, 511, 526, 557, 519, 535, 487, 517, 504, 487, 494, 538, 563, 518, 470, 515, 533, 544, 507, 477, 561] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[497, 531, 520, 523, 524, 493, 520, 547, 530, 531, 508, 547, 497, 499, 522, 501, 510, 507, 523, 529, 499, 509, 526, 497, 522, 490, 543, 481, 533, 505, 516, 483, 505, 495, 501, 502, 510, 543, 503, 550, 543, 493, 519, 503, 510, 509, 485, 505, 516, 505, 523, 485, 546, 531, 532, 469, 510, 510, 515, 489, 529, 488, 489, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[520, 515, 521, 535, 541, 512, 490, 506, 499, 544, 498, 484, 510, 548, 559, 486, 514, 474, 514, 525, 508, 511, 514, 504, 496, 538, 530, 560, 506, 536, 499, 525, 505, 492, 519, 503, 504, 521, 501, 501, 512, 507, 548, 449, 532, 498, 502, 537, 501, 497, 523, 485, 494, 527, 519, 529, 467, 510, 517, 502, 494, 519, 517, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[524, 498, 509, 515, 487, 506, 492, 544, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 503, 494, 494, 488, 481, 498, 6, 5, 0, 0, 0, 0, 0, 0, 492, 528, 540, 513, 478, 526, 486, 493, 11, 9, 0, 0, 0, 0, 0, 0, 546, 480, 521, 529, 467, 513, 537, 532, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 502, 522, 520, 514, 517, 533, 0, 0, 0, 0, 0, 0, 0, 0, 554, 499, 517, 532, 540, 501, 551, 493, 5, 0, 0, 0, 0, 0, 0, 0, 516, 479, 532, 497, 515, 526, 500, 500, 18, 0, 0, 0, 0, 0, 0, 0, 526, 536, 520, 518, 518, 498, 465, 490, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 3994, 4076, 4125, 4124, 4192, 4083, 4099] avg=4096.0000 max=4192 max_rank=5 min=3994 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[485, 547, 516, 516, 517, 523, 536, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 536, 499, 491, 558, 494, 480, 527, 6, 5, 0, 0, 0, 0, 0, 0, 525, 532, 488, 478, 547, 497, 513, 520, 12, 9, 0, 0, 0, 0, 0, 0, 503, 506, 514, 513, 522, 513, 484, 535, 0, 0, 0, 0, 0, 0, 0, 0, 488, 511, 530, 496, 491, 517, 476, 492, 0, 0, 0, 0, 0, 0, 0, 0, 503, 541, 520, 522, 540, 510, 505, 528, 5, 0, 0, 0, 0, 0, 0, 0, 495, 512, 526, 505, 505, 513, 464, 511, 17, 0, 0, 0, 0, 0, 0, 0, 480, 507, 541, 467, 500, 526, 511, 517, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4085, 4121, 4090, 4001, 4174, 4048, 4077] avg=4096.0000 max=4174 max_rank=5 min=4001 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[506, 518, 508, 465, 530, 485, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 515, 527, 499, 532, 506, 518, 6, 4, 0, 0, 0, 0, 0, 0, 513, 516, 500, 490, 507, 562, 491, 532, 10, 8, 0, 0, 0, 0, 0, 0, 520, 533, 525, 479, 520, 500, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 535, 529, 493, 530, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 519, 529, 510, 490, 529, 540, 498, 495, 5, 0, 0, 0, 0, 0, 0, 0, 522, 507, 456, 546, 522, 481, 500, 514, 16, 0, 0, 0, 0, 0, 0, 0, 526, 494, 479, 503, 511, 521, 504, 506, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4102, 4129, 4084, 4160, 4115, 4064, 4069] avg=4096.0000 max=4160 max_rank=4 min=4045 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[541, 496, 456, 458, 519, 544, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 491, 545, 514, 496, 512, 503, 9, 9, 0, 0, 0, 0, 0, 0, 545, 527, 491, 505, 487, 514, 517, 461, 14, 12, 0, 0, 0, 0, 0, 0, 526, 514, 523, 525, 529, 466, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 488, 493, 511, 473, 573, 535, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 507, 516, 486, 514, 494, 533, 11, 0, 0, 0, 0, 0, 0, 0, 496, 512, 518, 524, 537, 506, 493, 515, 21, 0, 0, 0, 0, 0, 0, 0, 497, 495, 527, 520, 506, 518, 481, 527, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4111, 4073, 4125, 4097, 4074, 4122, 4101] avg=4096.0000 max=4125 max_rank=3 min=4065 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 535, 533, 485, 554, 550, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 520, 545, 521, 522, 494, 518, 6, 4, 0, 0, 0, 0, 0, 0, 538, 507, 522, 489, 493, 546, 504, 523, 11, 8, 0, 0, 0, 0, 0, 0, 519, 530, 507, 456, 506, 514, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 506, 473, 512, 531, 512, 523, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 503, 473, 479, 456, 486, 517, 511, 499, 5, 0, 0, 0, 0, 0, 0, 0, 533, 536, 485, 478, 518, 535, 497, 498, 15, 0, 0, 0, 0, 0, 0, 0, 507, 512, 534, 525, 486, 500, 549, 514, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4137, 4141, 4034, 4102, 3929, 4095, 4154] avg=4096.0000 max=4176 max_rank=0 min=3929 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 523, 471, 484, 487, 524, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 475, 504, 519, 510, 504, 496, 540, 496, 6, 5, 0, 0, 0, 0, 0, 0, 499, 513, 500, 516, 478, 478, 532, 491, 11, 8, 0, 0, 0, 0, 0, 0, 516, 544, 509, 491, 492, 509, 525, 519, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 500, 537, 566, 492, 522, 511, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 516, 466, 511, 526, 557, 519, 5, 0, 0, 0, 0, 0, 0, 0, 535, 487, 517, 504, 487, 494, 538, 563, 16, 0, 0, 0, 0, 0, 0, 0, 518, 470, 515, 533, 544, 507, 477, 561, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4055, 4026, 4105, 4151, 4122, 4141, 4152] avg=4096.0000 max=4152 max_rank=7 min=4016 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[497, 531, 520, 485, 524, 493, 520, 547, 0, 0, 0, 0, 0, 0, 0, 0, 530, 531, 508, 547, 497, 499, 522, 501, 6, 4, 0, 0, 0, 0, 0, 0, 510, 507, 523, 529, 499, 509, 526, 497, 11, 7, 0, 0, 0, 0, 0, 0, 522, 490, 543, 455, 533, 505, 516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 501, 502, 510, 532, 503, 550, 0, 0, 0, 0, 0, 0, 0, 0, 543, 493, 519, 503, 510, 509, 485, 505, 5, 0, 0, 0, 0, 0, 0, 0, 516, 505, 523, 485, 546, 531, 532, 469, 15, 0, 0, 0, 0, 0, 0, 0, 510, 510, 515, 489, 529, 488, 489, 492, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4145, 4118, 4047, 4098, 4072, 4122, 4049] avg=4096.0000 max=4145 max_rank=1 min=4047 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 515, 521, 497, 541, 512, 490, 506, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 498, 484, 510, 548, 559, 486, 6, 5, 0, 0, 0, 0, 0, 0, 514, 474, 514, 525, 508, 511, 514, 504, 11, 9, 0, 0, 0, 0, 0, 0, 496, 538, 530, 528, 506, 536, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 519, 503, 504, 510, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 507, 548, 449, 532, 498, 502, 537, 5, 0, 0, 0, 0, 0, 0, 0, 501, 497, 523, 485, 494, 527, 519, 529, 18, 0, 0, 0, 0, 0, 0, 0, 467, 510, 517, 502, 494, 519, 517, 514, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4139, 4084, 4158, 4035, 4090, 4093, 4067] avg=4096.0000 max=4158 max_rank=3 min=4035 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[516, 504, 480, 496, 499, 539, 528, 515, 548, 509, 522, 518, 513, 528, 513, 541, 530, 526, 513, 525, 497, 543, 511, 495, 493, 491, 517, 493, 509, 559, 517, 506, 511, 474, 500, 495, 508, 516, 535, 530, 484, 509, 532, 492, 500, 463, 477, 524, 525, 493, 475, 547, 515, 508, 505, 475, 548, 498, 525, 520, 508, 530, 518, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[504, 515, 533, 457, 489, 495, 525, 505, 541, 515, 513, 478, 521, 515, 492, 500, 532, 475, 500, 508, 509, 492, 512, 517, 524, 547, 492, 502, 511, 521, 520, 534, 513, 492, 509, 532, 525, 495, 506, 509, 527, 512, 524, 542, 537, 534, 511, 554, 497, 468, 532, 556, 500, 487, 506, 556, 511, 527, 501, 539, 470, 519, 493, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[514, 512, 516, 516, 534, 491, 511, 517, 535, 487, 514, 470, 518, 515, 513, 480, 500, 470, 508, 516, 488, 517, 518, 567, 566, 573, 518, 522, 514, 496, 476, 509, 519, 507, 537, 508, 532, 511, 536, 502, 482, 523, 490, 511, 518, 511, 515, 507, 527, 488, 526, 526, 492, 474, 507, 554, 483, 538, 528, 523, 525, 493, 499, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[525, 501, 514, 531, 474, 500, 476, 517, 515, 546, 517, 510, 474, 493, 491, 486, 506, 460, 512, 514, 522, 523, 482, 473, 553, 516, 516, 505, 516, 557, 490, 514, 507, 535, 551, 511, 534, 554, 494, 491, 516, 511, 546, 498, 525, 519, 540, 551, 533, 515, 503, 492, 467, 515, 491, 523, 494, 480, 537, 556, 495, 516, 516, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[507, 533, 519, 512, 521, 541, 519, 523, 484, 495, 528, 535, 508, 488, 500, 533, 538, 523, 525, 552, 465, 514, 478, 489, 503, 511, 492, 485, 499, 474, 538, 517, 474, 533, 557, 502, 521, 521, 560, 539, 468, 480, 531, 508, 480, 506, 511, 564, 553, 451, 538, 500, 472, 558, 503, 484, 490, 533, 533, 496, 490, 537, 488, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[480, 507, 498, 519, 502, 525, 510, 489, 495, 505, 515, 501, 513, 545, 531, 523, 489, 518, 535, 549, 571, 508, 523, 507, 513, 516, 502, 538, 526, 500, 521, 500, 496, 505, 534, 498, 499, 467, 531, 529, 504, 497, 521, 541, 478, 518, 513, 504, 511, 532, 497, 528, 505, 452, 526, 495, 519, 499, 503, 530, 475, 506, 550, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[511, 513, 536, 546, 535, 525, 528, 541, 521, 497, 547, 503, 514, 498, 479, 540, 540, 460, 507, 508, 522, 555, 475, 517, 486, 503, 501, 482, 508, 521, 513, 529, 526, 497, 510, 506, 506, 501, 527, 512, 486, 489, 522, 508, 540, 535, 477, 503, 460, 504, 515, 484, 538, 526, 493, 520, 565, 537, 489, 517, 498, 520, 489, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[490, 497, 521, 509, 514, 497, 525, 499, 487, 526, 485, 547, 518, 536, 522, 520, 503, 511, 502, 481, 538, 492, 517, 496, 524, 530, 528, 501, 532, 518, 488, 510, 551, 508, 520, 526, 479, 498, 530, 504, 494, 501, 551, 518, 510, 471, 506, 543, 509, 490, 536, 505, 505, 554, 516, 509, 493, 535, 493, 532, 521, 519, 473, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 512, 516, 516, 534, 491, 511, 517, 3, 0, 0, 0, 0, 0, 0, 0, 535, 487, 514, 470, 518, 515, 513, 480, 0, 0, 0, 0, 0, 0, 0, 0, 500, 470, 508, 516, 488, 517, 518, 567, 10, 2, 0, 0, 0, 0, 0, 0, 566, 558, 518, 522, 514, 496, 476, 509, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 537, 508, 532, 511, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 482, 523, 490, 511, 518, 511, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 526, 526, 492, 474, 507, 554, 22, 5, 0, 0, 0, 0, 0, 0, 483, 538, 528, 518, 525, 493, 499, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4032, 4096, 4159, 4130, 4057, 4121, 4059] avg=4096.0000 max=4159 max_rank=3 min=4032 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 504, 480, 496, 499, 539, 528, 515, 3, 0, 0, 0, 0, 0, 0, 0, 548, 509, 522, 518, 513, 528, 513, 541, 0, 0, 0, 0, 0, 0, 0, 0, 530, 526, 513, 525, 497, 543, 511, 495, 8, 2, 0, 0, 0, 0, 0, 0, 493, 478, 517, 493, 509, 559, 517, 506, 0, 0, 0, 0, 0, 0, 0, 0, 511, 474, 500, 495, 508, 516, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 532, 492, 500, 463, 477, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 493, 475, 547, 515, 508, 505, 475, 22, 5, 0, 0, 0, 0, 0, 0, 548, 498, 525, 515, 508, 530, 518, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4192, 4150, 4072, 4047, 3981, 4070, 4176] avg=4096.0000 max=4192 max_rank=1 min=3981 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[504, 515, 533, 457, 489, 495, 525, 505, 3, 0, 0, 0, 0, 0, 0, 0, 541, 515, 513, 478, 521, 515, 492, 500, 0, 0, 0, 0, 0, 0, 0, 0, 532, 475, 500, 508, 509, 492, 512, 517, 9, 3, 0, 0, 0, 0, 0, 0, 524, 533, 492, 502, 511, 521, 520, 534, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 509, 532, 525, 495, 485, 509, 0, 0, 0, 0, 0, 0, 0, 0, 527, 512, 524, 542, 537, 534, 511, 554, 0, 0, 0, 0, 0, 0, 0, 0, 497, 468, 532, 556, 500, 487, 506, 556, 21, 5, 0, 0, 0, 0, 0, 0, 511, 527, 501, 533, 470, 519, 493, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4075, 4057, 4137, 4060, 4241, 4128, 4044] avg=4096.0000 max=4241 max_rank=5 min=4026 min_rank=0 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 501, 514, 531, 474, 500, 476, 517, 3, 0, 0, 0, 0, 0, 0, 0, 515, 546, 517, 510, 474, 493, 491, 486, 0, 0, 0, 0, 0, 0, 0, 0, 506, 460, 512, 514, 522, 523, 482, 473, 9, 3, 0, 0, 0, 0, 0, 0, 553, 502, 516, 505, 516, 557, 490, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 535, 551, 511, 534, 554, 474, 491, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 546, 498, 525, 519, 540, 551, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 503, 492, 467, 515, 491, 523, 20, 5, 0, 0, 0, 0, 0, 0, 494, 480, 537, 550, 495, 516, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4032, 4004, 4153, 4157, 4206, 4064, 4111] avg=4096.0000 max=4206 max_rank=5 min=4004 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[480, 507, 498, 519, 502, 525, 510, 489, 3, 0, 0, 0, 0, 0, 0, 0, 495, 505, 515, 501, 513, 545, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 489, 518, 535, 549, 571, 508, 523, 507, 9, 3, 0, 0, 0, 0, 0, 0, 513, 502, 502, 538, 526, 500, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 505, 534, 498, 499, 467, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 521, 541, 478, 518, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 497, 528, 505, 452, 526, 495, 22, 5, 0, 0, 0, 0, 0, 0, 519, 499, 503, 524, 475, 506, 550, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4128, 4212, 4102, 4037, 4076, 4073, 4107] avg=4096.0000 max=4212 max_rank=2 min=4033 min_rank=0 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[507, 533, 519, 512, 521, 541, 519, 523, 3, 0, 0, 0, 0, 0, 0, 0, 484, 495, 528, 535, 508, 488, 500, 533, 0, 0, 0, 0, 0, 0, 0, 0, 538, 523, 525, 552, 465, 514, 478, 489, 9, 2, 0, 0, 0, 0, 0, 0, 503, 497, 492, 485, 499, 474, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 474, 533, 557, 502, 521, 521, 537, 539, 0, 0, 0, 0, 0, 0, 0, 0, 468, 480, 531, 508, 480, 506, 511, 564, 0, 0, 0, 0, 0, 0, 0, 0, 553, 451, 538, 500, 472, 558, 503, 484, 23, 5, 0, 0, 0, 0, 0, 0, 490, 533, 533, 491, 490, 537, 488, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4071, 4095, 4005, 4184, 4048, 4087, 4100] avg=4096.0000 max=4184 max_rank=4 min=4005 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 513, 536, 546, 535, 525, 528, 541, 6, 0, 0, 0, 0, 0, 0, 0, 521, 497, 545, 503, 514, 498, 479, 540, 0, 0, 0, 0, 0, 0, 0, 0, 540, 460, 507, 508, 522, 555, 475, 517, 12, 6, 0, 0, 0, 0, 0, 0, 486, 483, 501, 482, 508, 521, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 526, 497, 510, 506, 506, 501, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 486, 489, 522, 508, 540, 535, 477, 503, 3, 2, 0, 0, 0, 0, 0, 0, 460, 504, 515, 484, 538, 526, 493, 520, 26, 8, 0, 0, 0, 0, 0, 0, 565, 537, 489, 502, 498, 520, 489, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4241, 4097, 4102, 4023, 4059, 4065, 4074, 4107] avg=4096.0000 max=4241 max_rank=0 min=4023 min_rank=3 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[490, 497, 521, 509, 514, 497, 525, 499, 3, 0, 0, 0, 0, 0, 0, 0, 487, 526, 485, 547, 518, 536, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 502, 481, 538, 492, 517, 496, 9, 3, 0, 0, 0, 0, 0, 0, 524, 516, 528, 501, 532, 518, 488, 510, 0, 0, 0, 0, 0, 0, 0, 0, 551, 508, 520, 526, 479, 498, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 501, 551, 518, 510, 471, 506, 543, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 536, 505, 505, 554, 516, 509, 22, 5, 0, 0, 0, 0, 0, 0, 493, 535, 493, 526, 521, 519, 473, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4141, 4052, 4117, 4094, 4094, 4151, 4064] avg=4096.0000 max=4151 max_rank=6 min=4052 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4096, 4352, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[523, 528, 497, 529, 513, 535, 518, 474, 540, 528, 534, 495, 518, 478, 538, 503, 501, 504, 511, 517, 507, 525, 528, 572, 509, 514, 538, 478, 531, 523, 511, 496, 458, 501, 494, 480, 512, 501, 477, 496, 539, 514, 510, 525, 533, 500, 477, 502, 517, 539, 519, 500, 500, 532, 527, 512, 514, 531, 486, 493, 505, 522, 512, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[564, 496, 529, 502, 503, 521, 525, 494, 503, 507, 502, 541, 505, 522, 492, 527, 532, 523, 528, 487, 506, 551, 533, 515, 511, 512, 488, 501, 488, 501, 538, 500, 507, 516, 515, 493, 475, 496, 491, 509, 491, 504, 523, 536, 515, 503, 494, 503, 510, 535, 517, 550, 505, 506, 528, 506, 506, 503, 504, 510, 506, 486, 541, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[479, 549, 534, 565, 482, 518, 518, 532, 488, 523, 496, 483, 506, 521, 509, 552, 492, 529, 495, 494, 517, 500, 552, 528, 536, 522, 493, 518, 510, 503, 511, 517, 525, 555, 500, 484, 488, 499, 492, 524, 491, 531, 531, 529, 508, 496, 492, 545, 507, 469, 499, 526, 502, 532, 521, 503, 526, 499, 498, 511, 511, 533, 473, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[525, 519, 515, 494, 507, 574, 477, 538, 471, 531, 486, 495, 463, 520, 506, 549, 473, 505, 528, 499, 530, 473, 485, 504, 528, 479, 516, 491, 533, 529, 502, 570, 493, 524, 525, 510, 489, 488, 530, 513, 501, 509, 501, 529, 497, 513, 523, 540, 534, 520, 509, 509, 499, 493, 539, 534, 507, 518, 541, 514, 521, 545, 473, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[512, 488, 507, 501, 532, 485, 520, 517, 530, 481, 526, 495, 520, 512, 479, 491, 479, 535, 536, 516, 541, 494, 541, 542, 540, 490, 563, 527, 552, 494, 504, 496, 493, 513, 531, 487, 485, 511, 528, 513, 540, 533, 509, 526, 493, 515, 503, 528, 511, 501, 545, 491, 512, 532, 477, 504, 513, 481, 512, 500, 517, 538, 508, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[484, 512, 494, 499, 480, 509, 462, 496, 545, 572, 535, 500, 507, 532, 535, 507, 516, 538, 523, 549, 526, 530, 534, 510, 530, 506, 529, 509, 539, 477, 481, 514, 547, 536, 512, 510, 487, 510, 501, 503, 523, 518, 502, 491, 488, 473, 492, 526, 508, 512, 557, 510, 525, 505, 492, 507, 499, 509, 504, 486, 482, 511, 547, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[550, 503, 514, 482, 511, 497, 542, 539, 516, 534, 513, 519, 536, 517, 521, 466, 539, 529, 493, 515, 495, 530, 546, 485, 546, 495, 494, 519, 506, 509, 481, 480, 496, 478, 581, 515, 493, 488, 524, 513, 476, 542, 491, 531, 536, 508, 516, 457, 486, 504, 474, 517, 532, 545, 511, 521, 514, 501, 512, 502, 539, 508, 515, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[472, 523, 527, 487, 531, 456, 556, 495, 522, 533, 484, 503, 535, 521, 503, 494, 502, 525, 507, 534, 514, 516, 464, 509, 502, 537, 515, 491, 518, 502, 520, 533, 512, 500, 525, 489, 509, 535, 538, 514, 495, 524, 527, 538, 516, 500, 510, 478, 484, 540, 537, 524, 545, 504, 500, 493, 491, 548, 520, 535, 514, 493, 517, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[564, 496, 529, 502, 503, 521, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 502, 541, 505, 522, 492, 527, 0, 0, 0, 0, 0, 0, 0, 0, 532, 487, 528, 487, 506, 551, 533, 515, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 488, 501, 488, 501, 538, 500, 0, 0, 0, 0, 0, 0, 0, 0, 507, 516, 515, 493, 475, 496, 491, 509, 36, 5, 0, 0, 0, 0, 0, 0, 491, 504, 523, 536, 515, 503, 494, 503, 4, 2, 0, 0, 0, 0, 0, 0, 510, 535, 501, 550, 505, 506, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 504, 510, 506, 486, 541, 537, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4094, 4139, 4037, 4043, 4075, 4141, 4105] avg=4096.0000 max=4141 max_rank=6 min=4037 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 519, 515, 494, 507, 574, 477, 538, 1, 0, 0, 0, 0, 0, 0, 0, 471, 526, 486, 495, 463, 520, 506, 549, 0, 0, 0, 0, 0, 0, 0, 0, 473, 471, 528, 499, 530, 473, 485, 504, 0, 0, 0, 0, 0, 0, 0, 0, 525, 479, 516, 491, 533, 529, 502, 570, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 525, 510, 489, 488, 530, 513, 34, 5, 0, 0, 0, 0, 0, 0, 501, 509, 501, 529, 497, 513, 523, 540, 4, 2, 0, 0, 0, 0, 0, 0, 534, 520, 493, 509, 499, 493, 539, 534, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 541, 514, 521, 545, 473, 512, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4016, 3963, 4145, 4111, 4119, 4121, 4143] avg=4096.0000 max=4150 max_rank=0 min=3963 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[523, 528, 497, 529, 513, 535, 518, 474, 0, 0, 0, 0, 0, 0, 0, 0, 540, 523, 534, 495, 518, 478, 538, 503, 0, 0, 0, 0, 0, 0, 0, 0, 501, 470, 511, 517, 507, 525, 528, 572, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 538, 478, 531, 523, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 494, 480, 512, 501, 477, 496, 34, 5, 0, 0, 0, 0, 0, 0, 539, 514, 510, 525, 533, 500, 477, 502, 4, 2, 0, 0, 0, 0, 0, 0, 517, 539, 503, 500, 500, 532, 527, 512, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 486, 493, 505, 522, 512, 524, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4129, 4131, 4098, 3958, 4106, 4130, 4099] avg=4096.0000 max=4131 max_rank=2 min=3958 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 488, 507, 501, 532, 485, 520, 517, 1, 0, 0, 0, 0, 0, 0, 0, 530, 476, 526, 495, 520, 512, 479, 491, 0, 0, 0, 0, 0, 0, 0, 0, 479, 499, 536, 516, 541, 494, 541, 542, 0, 0, 0, 0, 0, 0, 0, 0, 537, 490, 563, 527, 552, 494, 504, 496, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 531, 487, 485, 511, 528, 513, 36, 5, 0, 0, 0, 0, 0, 0, 540, 533, 509, 526, 493, 515, 503, 528, 4, 2, 0, 0, 0, 0, 0, 0, 511, 501, 528, 491, 512, 532, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0, 513, 481, 512, 500, 517, 538, 508, 472, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4029, 4148, 4163, 4102, 4153, 4056, 4054] avg=4096.0000 max=4163 max_rank=3 min=4029 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 549, 534, 565, 482, 518, 518, 532, 1, 0, 0, 0, 0, 0, 0, 0, 488, 518, 496, 483, 506, 521, 509, 552, 0, 0, 0, 0, 0, 0, 0, 0, 492, 493, 495, 494, 517, 500, 552, 528, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 493, 518, 510, 503, 511, 517, 0, 0, 0, 0, 0, 0, 0, 0, 525, 555, 500, 484, 488, 499, 492, 524, 36, 5, 0, 0, 0, 0, 0, 0, 491, 531, 531, 529, 508, 496, 492, 545, 4, 2, 0, 0, 0, 0, 0, 0, 507, 469, 483, 526, 502, 532, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 526, 499, 498, 511, 511, 533, 473, 496, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4073, 4071, 4107, 4108, 4129, 4043, 4059] avg=4096.0000 max=4178 max_rank=0 min=4043 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[550, 503, 514, 482, 511, 497, 542, 539, 4, 0, 0, 0, 0, 0, 0, 0, 516, 522, 513, 519, 536, 517, 521, 466, 0, 0, 0, 0, 0, 0, 0, 0, 539, 489, 493, 515, 495, 530, 546, 485, 0, 0, 0, 0, 0, 0, 0, 0, 536, 495, 494, 519, 506, 509, 481, 480, 0, 0, 0, 0, 0, 0, 0, 0, 496, 478, 581, 515, 493, 488, 524, 513, 40, 7, 0, 0, 0, 0, 0, 0, 476, 542, 491, 531, 536, 508, 516, 457, 9, 6, 0, 0, 0, 0, 0, 0, 486, 504, 452, 517, 532, 545, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 512, 502, 539, 508, 515, 520, 15, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4110, 4092, 4020, 4135, 4072, 4068, 4129] avg=4096.0000 max=4142 max_rank=0 min=4020 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[484, 512, 494, 499, 480, 509, 462, 496, 1, 0, 0, 0, 0, 0, 0, 0, 545, 566, 535, 500, 507, 532, 535, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 501, 523, 549, 526, 530, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 529, 509, 539, 477, 481, 514, 0, 0, 0, 0, 0, 0, 0, 0, 547, 536, 512, 510, 487, 510, 501, 503, 37, 5, 0, 0, 0, 0, 0, 0, 523, 518, 502, 491, 488, 473, 492, 526, 5, 2, 0, 0, 0, 0, 0, 0, 508, 512, 540, 510, 525, 505, 492, 507, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 504, 486, 482, 511, 547, 515, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3937, 4227, 4189, 4082, 4148, 4020, 4099, 4066] avg=4096.0000 max=4227 max_rank=1 min=3937 min_rank=0 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[472, 523, 527, 487, 531, 456, 556, 495, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 484, 503, 535, 521, 503, 494, 0, 0, 0, 0, 0, 0, 0, 0, 502, 489, 507, 534, 514, 516, 464, 509, 0, 0, 0, 0, 0, 0, 0, 0, 500, 537, 515, 491, 518, 502, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 525, 489, 509, 535, 538, 514, 36, 5, 0, 0, 0, 0, 0, 0, 495, 524, 527, 538, 516, 500, 510, 478, 4, 2, 0, 0, 0, 0, 0, 0, 484, 540, 520, 524, 545, 504, 500, 493, 0, 0, 0, 0, 0, 0, 0, 0, 491, 548, 520, 535, 514, 493, 517, 482, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4090, 4035, 4116, 4163, 4094, 4110, 4113] avg=4096.0000 max=4163 max_rank=4 min=4035 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 3968, 3968, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[488, 514, 476, 514, 532, 526, 510, 510, 538, 517, 502, 573, 456, 566, 520, 533, 498, 529, 511, 560, 530, 529, 506, 503, 510, 506, 518, 502, 523, 530, 540, 500, 517, 503, 530, 508, 473, 503, 498, 533, 467, 489, 483, 517, 522, 492, 514, 495, 521, 494, 504, 517, 496, 498, 542, 521, 512, 522, 517, 501, 508, 498, 485, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[536, 528, 512, 485, 496, 515, 541, 530, 495, 529, 564, 460, 496, 491, 522, 539, 511, 503, 480, 536, 507, 520, 518, 478, 518, 535, 535, 495, 507, 526, 522, 535, 511, 519, 462, 581, 478, 540, 497, 515, 542, 481, 508, 469, 553, 527, 520, 514, 508, 504, 503, 522, 480, 535, 485, 514, 532, 484, 502, 502, 478, 514, 526, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[467, 514, 489, 536, 574, 512, 504, 505, 507, 542, 517, 495, 494, 517, 495, 521, 461, 493, 507, 538, 538, 481, 500, 536, 495, 502, 505, 499, 538, 525, 500, 519, 538, 511, 491, 524, 526, 479, 497, 500, 516, 515, 531, 490, 509, 512, 489, 537, 487, 518, 516, 469, 518, 532, 502, 544, 528, 536, 475, 539, 492, 527, 577, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[492, 476, 525, 477, 519, 475, 545, 549, 547, 525, 521, 536, 507, 522, 532, 527, 527, 546, 506, 518, 531, 500, 519, 498, 523, 491, 517, 515, 520, 517, 549, 479, 511, 549, 510, 517, 510, 518, 455, 490, 518, 493, 508, 537, 536, 491, 484, 502, 499, 511, 510, 476, 522, 533, 527, 479, 507, 490, 512, 527, 499, 507, 525, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[552, 504, 476, 570, 508, 472, 492, 491, 516, 491, 533, 501, 486, 517, 541, 506, 548, 518, 513, 505, 556, 513, 519, 518, 440, 501, 507, 506, 508, 559, 504, 519, 510, 544, 520, 519, 543, 555, 504, 527, 493, 488, 480, 524, 525, 520, 524, 488, 511, 510, 493, 488, 556, 509, 472, 472, 546, 506, 543, 489, 526, 504, 480, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[505, 496, 514, 500, 531, 528, 523, 483, 482, 532, 513, 508, 483, 490, 541, 523, 535, 506, 543, 480, 513, 476, 508, 567, 497, 514, 515, 520, 483, 504, 511, 552, 526, 516, 493, 524, 535, 545, 490, 498, 499, 527, 518, 534, 516, 513, 507, 475, 506, 516, 484, 506, 506, 567, 517, 486, 500, 487, 535, 544, 506, 504, 500, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[477, 480, 504, 535, 539, 514, 518, 503, 513, 471, 535, 529, 526, 493, 516, 509, 467, 512, 486, 516, 501, 511, 482, 500, 509, 511, 542, 529, 491, 496, 491, 495, 509, 524, 544, 557, 503, 497, 509, 543, 491, 502, 525, 537, 536, 538, 536, 536, 522, 508, 524, 525, 503, 498, 478, 524, 483, 529, 515, 522, 496, 519, 529, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[535, 504, 532, 515, 506, 498, 512, 489, 503, 490, 525, 484, 524, 518, 521, 485, 538, 550, 471, 502, 507, 519, 473, 496, 499, 532, 523, 540, 519, 509, 515, 558, 518, 524, 499, 515, 473, 498, 515, 521, 506, 522, 538, 514, 498, 486, 559, 529, 532, 523, 535, 507, 526, 517, 522, 450, 502, 502, 524, 489, 487, 523, 506, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[536, 528, 512, 485, 496, 515, 541, 530, 12, 0, 0, 0, 0, 0, 0, 0, 495, 529, 545, 460, 496, 491, 522, 539, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 480, 536, 507, 520, 518, 478, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 535, 495, 507, 511, 522, 535, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 462, 565, 478, 540, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 542, 481, 508, 469, 553, 527, 520, 514, 3, 0, 0, 0, 0, 0, 0, 0, 508, 504, 503, 522, 480, 535, 485, 514, 19, 5, 0, 0, 0, 0, 0, 0, 532, 484, 502, 502, 478, 514, 526, 497, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4077, 4053, 4158, 4087, 4117, 4075, 4046] avg=4096.0000 max=4158 max_rank=3 min=4046 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[488, 514, 476, 514, 532, 526, 510, 510, 10, 0, 0, 0, 0, 0, 0, 0, 538, 517, 485, 573, 456, 566, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 498, 529, 511, 560, 530, 529, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 518, 502, 523, 515, 540, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 530, 495, 473, 503, 498, 533, 0, 0, 0, 0, 0, 0, 0, 0, 467, 489, 483, 517, 522, 492, 514, 495, 2, 0, 0, 0, 0, 0, 0, 0, 521, 494, 504, 517, 496, 498, 542, 521, 17, 5, 0, 0, 0, 0, 0, 0, 512, 522, 517, 501, 508, 498, 485, 518, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4188, 4166, 4114, 4052, 3981, 4115, 4072] avg=4096.0000 max=4188 max_rank=1 min=3981 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[492, 476, 525, 477, 519, 475, 545, 549, 10, 0, 0, 0, 0, 0, 0, 0, 547, 525, 504, 536, 507, 522, 532, 527, 0, 0, 0, 0, 0, 0, 0, 0, 527, 546, 506, 518, 531, 500, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 517, 515, 520, 502, 549, 479, 0, 0, 0, 0, 0, 0, 0, 0, 511, 549, 510, 504, 510, 518, 455, 490, 0, 0, 0, 0, 0, 0, 0, 0, 518, 493, 508, 537, 536, 491, 484, 502, 2, 0, 0, 0, 0, 0, 0, 0, 499, 511, 510, 476, 522, 533, 527, 479, 17, 5, 0, 0, 0, 0, 0, 0, 507, 490, 512, 527, 499, 507, 525, 484, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4200, 4145, 4096, 4047, 4071, 4079, 4062] avg=4096.0000 max=4200 max_rank=1 min=4047 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[467, 514, 489, 536, 574, 512, 504, 505, 11, 0, 0, 0, 0, 0, 0, 0, 507, 542, 500, 495, 494, 517, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 461, 493, 507, 538, 538, 481, 500, 536, 0, 0, 0, 0, 0, 0, 0, 0, 495, 502, 505, 499, 538, 510, 500, 519, 0, 0, 0, 0, 0, 0, 0, 0, 538, 511, 491, 510, 526, 479, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 531, 490, 509, 512, 489, 537, 2, 0, 0, 0, 0, 0, 0, 0, 487, 518, 516, 469, 518, 532, 502, 544, 17, 5, 0, 0, 0, 0, 0, 0, 528, 536, 475, 539, 492, 527, 577, 517, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4071, 4054, 4068, 4052, 4101, 4108, 4202] avg=4096.0000 max=4202 max_rank=7 min=4052 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[477, 480, 504, 535, 539, 514, 518, 503, 11, 0, 0, 0, 0, 0, 0, 0, 513, 471, 517, 529, 526, 493, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 467, 512, 486, 516, 501, 511, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 542, 529, 491, 482, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 509, 524, 544, 542, 503, 497, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 525, 537, 536, 538, 536, 536, 3, 0, 0, 0, 0, 0, 0, 0, 522, 508, 524, 525, 503, 498, 478, 524, 18, 5, 0, 0, 0, 0, 0, 0, 483, 529, 515, 522, 496, 519, 529, 505, 9, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4074, 3975, 4050, 4171, 4204, 4105, 4108] avg=4096.0000 max=4204 max_rank=5 min=3975 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16588 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[552, 504, 476, 570, 508, 472, 492, 491, 15, 0, 0, 0, 0, 0, 0, 0, 516, 491, 512, 501, 486, 517, 541, 506, 0, 0, 0, 0, 0, 0, 0, 0, 548, 518, 513, 505, 556, 513, 519, 518, 1, 0, 0, 0, 0, 0, 0, 0, 440, 501, 507, 506, 508, 535, 504, 519, 0, 0, 0, 0, 0, 0, 0, 0, 510, 544, 520, 493, 543, 555, 504, 527, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 480, 524, 525, 520, 524, 488, 7, 0, 0, 0, 0, 0, 0, 0, 511, 510, 493, 488, 556, 509, 472, 472, 21, 10, 0, 0, 0, 0, 0, 0, 546, 506, 543, 489, 526, 504, 480, 509, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4070, 4191, 4020, 4196, 4049, 4042, 4120] avg=4096.0000 max=4196 max_rank=4 min=4020 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 496, 514, 500, 531, 528, 523, 483, 11, 0, 0, 0, 0, 0, 0, 0, 482, 532, 496, 508, 483, 490, 541, 523, 0, 0, 0, 0, 0, 0, 0, 0, 535, 506, 543, 480, 513, 476, 508, 567, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 515, 520, 483, 490, 511, 552, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 493, 510, 535, 545, 490, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 518, 534, 516, 513, 507, 475, 2, 0, 0, 0, 0, 0, 0, 0, 506, 516, 484, 506, 506, 567, 517, 486, 17, 5, 0, 0, 0, 0, 0, 0, 500, 487, 535, 544, 506, 504, 500, 512, 9, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4055, 4128, 4082, 4113, 4091, 4110, 4098] avg=4096.0000 max=4128 max_rank=2 min=4055 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[535, 504, 532, 515, 506, 498, 512, 489, 10, 0, 0, 0, 0, 0, 0, 0, 503, 490, 508, 484, 524, 518, 521, 485, 0, 0, 0, 0, 0, 0, 0, 0, 538, 550, 471, 502, 507, 519, 473, 496, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 523, 540, 519, 494, 515, 558, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 499, 502, 473, 498, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 522, 538, 514, 498, 486, 559, 529, 2, 0, 0, 0, 0, 0, 0, 0, 532, 523, 535, 507, 526, 517, 522, 450, 17, 5, 0, 0, 0, 0, 0, 0, 502, 502, 524, 489, 487, 523, 506, 516, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4033, 4056, 4180, 4050, 4154, 4134, 4060] avg=4096.0000 max=4180 max_rank=3 min=4033 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[518, 490, 504, 511, 505, 513, 515, 515, 492, 486, 559, 508, 516, 528, 529, 507, 500, 532, 511, 521, 524, 532, 515, 534, 500, 497, 490, 526, 522, 491, 518, 507, 523, 557, 538, 499, 514, 530, 501, 531, 498, 468, 511, 498, 514, 525, 513, 520, 513, 525, 504, 482, 523, 533, 516, 471, 489, 484, 532, 505, 490, 533, 517, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[496, 509, 496, 501, 517, 496, 498, 503, 489, 506, 529, 525, 528, 528, 533, 503, 545, 564, 510, 504, 474, 516, 477, 487, 488, 501, 568, 490, 510, 504, 489, 500, 520, 542, 466, 494, 525, 531, 534, 523, 519, 528, 489, 518, 516, 495, 498, 529, 515, 544, 502, 501, 532, 516, 509, 521, 538, 527, 484, 531, 527, 494, 499, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[505, 537, 459, 516, 463, 502, 480, 517, 531, 489, 510, 532, 523, 535, 504, 554, 505, 531, 497, 490, 502, 486, 499, 496, 496, 491, 511, 515, 537, 522, 520, 511, 516, 513, 529, 538, 512, 507, 548, 517, 499, 507, 537, 534, 469, 496, 493, 505, 517, 530, 543, 525, 500, 517, 515, 531, 493, 495, 541, 516, 496, 516, 517, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[524, 483, 509, 498, 514, 522, 497, 518, 520, 507, 532, 473, 538, 527, 531, 528, 494, 521, 492, 513, 500, 502, 493, 518, 531, 519, 537, 530, 496, 481, 537, 544, 519, 508, 519, 525, 466, 490, 542, 512, 476, 519, 502, 529, 483, 510, 548, 500, 507, 537, 492, 501, 524, 491, 517, 511, 521, 523, 479, 506, 518, 495, 547, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[501, 489, 550, 511, 537, 507, 495, 507, 492, 529, 543, 512, 523, 501, 515, 525, 504, 502, 539, 512, 522, 530, 517, 515, 525, 489, 493, 487, 518, 473, 533, 540, 515, 544, 488, 494, 521, 475, 511, 514, 521, 535, 497, 503, 503, 539, 500, 563, 525, 530, 481, 514, 505, 502, 515, 505, 520, 493, 494, 504, 500, 508, 488, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[561, 515, 501, 504, 490, 487, 513, 492, 531, 527, 525, 505, 492, 529, 507, 559, 515, 460, 528, 516, 515, 490, 513, 508, 532, 507, 521, 486, 522, 517, 532, 537, 452, 496, 525, 511, 517, 495, 497, 516, 482, 502, 476, 543, 523, 503, 526, 534, 538, 496, 493, 529, 505, 490, 511, 516, 530, 519, 533, 520, 519, 515, 473, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[524, 529, 525, 511, 557, 537, 566, 520, 542, 525, 513, 462, 530, 519, 547, 483, 506, 523, 508, 525, 477, 505, 530, 514, 521, 507, 518, 523, 473, 516, 540, 503, 468, 539, 501, 509, 520, 517, 509, 514, 499, 491, 517, 514, 518, 530, 475, 470, 534, 481, 569, 486, 487, 507, 508, 491, 481, 504, 483, 511, 512, 527, 513, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[514, 540, 545, 506, 553, 498, 466, 519, 510, 523, 495, 492, 530, 512, 547, 472, 543, 496, 504, 522, 501, 513, 518, 576, 513, 511, 519, 491, 513, 498, 478, 515, 495, 539, 537, 516, 489, 488, 467, 496, 497, 517, 533, 506, 505, 517, 529, 487, 505, 540, 509, 498, 498, 523, 537, 527, 443, 505, 519, 508, 513, 526, 535, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 509, 496, 501, 517, 496, 498, 503, 8, 0, 0, 0, 0, 0, 0, 0, 489, 506, 529, 525, 528, 528, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 545, 564, 510, 504, 474, 516, 477, 487, 4, 0, 0, 0, 0, 0, 0, 0, 488, 501, 568, 490, 510, 504, 489, 500, 3, 1, 0, 0, 0, 0, 0, 0, 520, 542, 466, 494, 525, 531, 534, 523, 0, 0, 0, 0, 0, 0, 0, 0, 519, 528, 489, 518, 516, 495, 498, 529, 21, 0, 0, 0, 0, 0, 0, 0, 515, 539, 502, 501, 532, 516, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 538, 527, 484, 531, 527, 494, 499, 517, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4098, 4081, 4054, 4135, 4113, 4135, 4128] avg=4096.0000 max=4135 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[524, 483, 509, 498, 514, 522, 497, 518, 8, 0, 0, 0, 0, 0, 0, 0, 520, 507, 532, 473, 538, 527, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 492, 513, 500, 502, 493, 518, 4, 0, 0, 0, 0, 0, 0, 0, 531, 519, 537, 530, 496, 481, 537, 544, 3, 1, 0, 0, 0, 0, 0, 0, 519, 508, 519, 525, 466, 490, 542, 512, 0, 0, 0, 0, 0, 0, 0, 0, 476, 519, 502, 529, 483, 510, 548, 500, 21, 0, 0, 0, 0, 0, 0, 0, 507, 532, 492, 501, 524, 491, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 521, 523, 479, 506, 518, 495, 547, 522, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4113, 4037, 4179, 4081, 4088, 4075, 4122] avg=4096.0000 max=4179 max_rank=3 min=4037 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 504, 511, 505, 513, 515, 515, 8, 0, 0, 0, 0, 0, 0, 0, 492, 486, 559, 508, 516, 528, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 511, 521, 524, 532, 515, 534, 4, 0, 0, 0, 0, 0, 0, 0, 500, 497, 490, 526, 522, 491, 518, 507, 3, 1, 0, 0, 0, 0, 0, 0, 523, 557, 538, 499, 514, 530, 501, 531, 0, 0, 0, 0, 0, 0, 0, 0, 498, 468, 511, 498, 514, 525, 513, 520, 20, 0, 0, 0, 0, 0, 0, 0, 513, 520, 504, 482, 523, 533, 516, 471, 0, 0, 0, 0, 0, 0, 0, 0, 489, 484, 532, 505, 490, 533, 517, 495, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4083, 4173, 4055, 4193, 4067, 4062, 4056] avg=4096.0000 max=4193 max_rank=4 min=4055 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[505, 537, 459, 516, 463, 502, 480, 517, 8, 0, 0, 0, 0, 0, 0, 0, 531, 489, 510, 532, 523, 535, 464, 554, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 497, 490, 502, 486, 499, 496, 4, 0, 0, 0, 0, 0, 0, 0, 496, 491, 511, 515, 537, 522, 520, 511, 3, 1, 0, 0, 0, 0, 0, 0, 516, 513, 529, 538, 512, 507, 548, 517, 0, 0, 0, 0, 0, 0, 0, 0, 499, 507, 537, 534, 469, 496, 493, 505, 19, 0, 0, 0, 0, 0, 0, 0, 517, 525, 543, 525, 500, 517, 515, 531, 0, 0, 0, 0, 0, 0, 0, 0, 493, 495, 541, 516, 496, 516, 517, 530, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4138, 4010, 4107, 4180, 4059, 4173, 4114] avg=4096.0000 max=4180 max_rank=4 min=3987 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[524, 529, 525, 511, 557, 537, 566, 520, 8, 0, 0, 0, 0, 0, 0, 0, 542, 525, 513, 462, 530, 519, 504, 483, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 508, 525, 477, 505, 530, 514, 4, 0, 0, 0, 0, 0, 0, 0, 521, 507, 518, 523, 473, 516, 540, 503, 3, 1, 0, 0, 0, 0, 0, 0, 468, 539, 501, 509, 520, 517, 509, 514, 0, 0, 0, 0, 0, 0, 0, 0, 499, 491, 517, 514, 518, 530, 475, 470, 21, 0, 0, 0, 0, 0, 0, 0, 534, 476, 569, 486, 487, 507, 508, 491, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 483, 511, 512, 527, 513, 504, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4277, 4078, 4092, 4105, 4077, 4035, 4058, 4046] avg=4096.0000 max=4277 max_rank=0 min=4035 min_rank=5 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[501, 489, 550, 511, 537, 507, 495, 507, 8, 0, 0, 0, 0, 0, 0, 0, 492, 529, 543, 512, 523, 501, 474, 525, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 539, 512, 522, 530, 517, 515, 4, 0, 0, 0, 0, 0, 0, 0, 525, 489, 493, 487, 518, 473, 533, 540, 3, 1, 0, 0, 0, 0, 0, 0, 515, 544, 488, 494, 521, 475, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 497, 503, 503, 539, 500, 563, 20, 0, 0, 0, 0, 0, 0, 0, 525, 525, 481, 514, 505, 502, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 494, 504, 500, 508, 488, 525, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4099, 4145, 4062, 4062, 4181, 4072, 4042] avg=4096.0000 max=4181 max_rank=5 min=4042 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[561, 515, 501, 504, 490, 487, 513, 492, 11, 0, 0, 0, 0, 0, 0, 0, 531, 527, 525, 505, 492, 529, 453, 559, 0, 0, 0, 0, 0, 0, 0, 0, 515, 460, 528, 516, 515, 490, 513, 508, 8, 0, 0, 0, 0, 0, 0, 0, 532, 507, 521, 486, 522, 517, 532, 537, 5, 5, 0, 0, 0, 0, 0, 0, 452, 496, 525, 511, 517, 495, 497, 516, 4, 0, 0, 0, 0, 0, 0, 0, 482, 502, 476, 543, 523, 503, 526, 534, 24, 0, 0, 0, 0, 0, 0, 0, 538, 479, 493, 529, 505, 490, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 533, 520, 519, 515, 473, 546, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4121, 4053, 4164, 4013, 4113, 4061, 4169] avg=4096.0000 max=4169 max_rank=7 min=4013 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 540, 545, 506, 553, 498, 466, 519, 8, 0, 0, 0, 0, 0, 0, 0, 510, 523, 495, 492, 530, 512, 504, 472, 0, 0, 0, 0, 0, 0, 0, 0, 543, 496, 504, 522, 501, 513, 518, 576, 4, 0, 0, 0, 0, 0, 0, 0, 513, 511, 519, 491, 513, 498, 478, 515, 3, 1, 0, 0, 0, 0, 0, 0, 495, 539, 537, 516, 489, 488, 467, 496, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 533, 506, 505, 517, 529, 487, 21, 0, 0, 0, 0, 0, 0, 0, 505, 535, 509, 498, 498, 523, 537, 527, 0, 0, 0, 0, 0, 0, 0, 0, 443, 505, 519, 508, 513, 526, 535, 531, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4038, 4177, 4042, 4027, 4112, 4132, 4091] avg=4096.0000 max=4177 max_rank=2 min=4027 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4352, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[493, 531, 509, 527, 525, 519, 516, 524, 479, 526, 512, 515, 519, 535, 513, 504, 527, 493, 485, 521, 502, 530, 523, 518, 528, 492, 484, 507, 499, 511, 531, 481, 512, 518, 494, 503, 525, 525, 505, 546, 522, 511, 505, 526, 535, 502, 520, 495, 512, 519, 515, 483, 490, 535, 511, 508, 502, 521, 543, 493, 513, 508, 476, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[476, 527, 523, 539, 520, 532, 510, 535, 493, 502, 508, 513, 505, 563, 498, 523, 507, 500, 501, 538, 507, 534, 520, 528, 507, 515, 535, 538, 509, 485, 542, 555, 502, 517, 517, 515, 495, 494, 519, 506, 520, 504, 497, 514, 514, 505, 492, 519, 503, 506, 529, 499, 498, 480, 471, 498, 524, 489, 504, 549, 479, 492, 489, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[519, 517, 541, 501, 537, 470, 497, 524, 475, 523, 483, 545, 490, 548, 504, 521, 495, 487, 509, 535, 545, 494, 520, 528, 528, 508, 559, 490, 496, 557, 543, 518, 517, 492, 467, 500, 513, 524, 531, 512, 510, 539, 478, 488, 507, 486, 523, 503, 485, 514, 504, 507, 515, 540, 496, 504, 514, 531, 530, 497, 521, 504, 499, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[531, 533, 561, 506, 520, 520, 519, 489, 528, 523, 519, 477, 502, 517, 501, 500, 508, 500, 525, 514, 502, 530, 497, 536, 518, 559, 511, 466, 510, 533, 508, 479, 518, 494, 469, 518, 536, 503, 510, 506, 516, 505, 520, 516, 551, 502, 537, 503, 515, 543, 508, 525, 473, 504, 480, 533, 519, 474, 517, 497, 485, 522, 498, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[521, 513, 478, 564, 511, 500, 509, 560, 512, 513, 538, 498, 488, 547, 530, 524, 554, 525, 525, 514, 506, 515, 515, 528, 512, 479, 531, 527, 492, 540, 521, 496, 498, 493, 485, 522, 508, 482, 483, 502, 532, 507, 543, 477, 519, 496, 473, 498, 508, 488, 498, 543, 513, 517, 536, 511, 545, 520, 486, 487, 501, 491, 508, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[466, 517, 508, 479, 501, 497, 520, 510, 545, 509, 499, 503, 501, 517, 501, 515, 526, 502, 483, 486, 516, 497, 513, 515, 513, 528, 541, 512, 520, 512, 550, 501, 450, 535, 555, 503, 538, 514, 516, 528, 510, 509, 460, 561, 487, 465, 510, 518, 494, 536, 533, 496, 538, 512, 552, 496, 506, 528, 545, 501, 538, 516, 495, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[508, 517, 495, 504, 479, 524, 506, 531, 504, 509, 507, 509, 539, 512, 494, 537, 525, 495, 511, 465, 451, 514, 540, 555, 543, 538, 517, 510, 511, 484, 499, 527, 515, 524, 500, 506, 490, 527, 502, 506, 492, 506, 493, 540, 520, 513, 506, 545, 494, 508, 520, 504, 532, 510, 475, 505, 508, 506, 528, 505, 505, 536, 531, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[510, 450, 510, 485, 510, 505, 505, 523, 538, 544, 543, 510, 523, 508, 490, 519, 490, 512, 517, 517, 491, 514, 502, 546, 499, 518, 505, 541, 522, 498, 522, 522, 484, 516, 551, 496, 491, 491, 533, 490, 501, 507, 524, 520, 505, 488, 516, 500, 494, 535, 512, 494, 564, 484, 541, 501, 507, 538, 524, 530, 537, 524, 506, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[493, 531, 509, 527, 525, 519, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 479, 526, 512, 515, 519, 520, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 527, 493, 485, 521, 502, 530, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 528, 492, 484, 507, 499, 511, 498, 481, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 494, 503, 525, 525, 505, 546, 29, 0, 0, 0, 0, 0, 0, 0, 522, 511, 505, 526, 535, 502, 520, 495, 15, 4, 0, 0, 0, 0, 0, 0, 512, 519, 515, 483, 490, 535, 511, 508, 6, 4, 4, 0, 0, 0, 0, 0, 502, 521, 539, 493, 513, 508, 476, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4088, 4093, 4000, 4157, 4135, 4087, 4068] avg=4096.0000 max=4157 max_rank=4 min=4000 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[476, 527, 523, 539, 520, 532, 510, 531, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 508, 513, 505, 547, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 501, 538, 507, 534, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 535, 538, 509, 485, 509, 555, 0, 0, 0, 0, 0, 0, 0, 0, 502, 517, 517, 515, 495, 494, 519, 506, 29, 0, 0, 0, 0, 0, 0, 0, 520, 504, 497, 514, 514, 505, 492, 519, 16, 4, 0, 0, 0, 0, 0, 0, 503, 506, 529, 499, 498, 480, 471, 498, 6, 4, 3, 0, 0, 0, 0, 0, 524, 489, 501, 549, 479, 492, 489, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4089, 4129, 4153, 4094, 4085, 3997, 4063] avg=4096.0000 max=4158 max_rank=0 min=3997 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 517, 541, 501, 537, 470, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 475, 523, 483, 545, 490, 533, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 495, 487, 509, 535, 545, 494, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 559, 490, 496, 557, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 467, 500, 513, 524, 531, 512, 29, 0, 0, 0, 0, 0, 0, 0, 510, 539, 478, 488, 507, 486, 523, 503, 15, 4, 0, 0, 0, 0, 0, 0, 485, 514, 504, 507, 515, 540, 496, 504, 6, 4, 4, 0, 0, 0, 0, 0, 514, 531, 526, 497, 521, 504, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4074, 4107, 4166, 4085, 4053, 4079, 4102] avg=4096.0000 max=4166 max_rank=3 min=4053 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 533, 561, 506, 520, 520, 519, 485, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 519, 477, 502, 503, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 525, 514, 502, 530, 497, 530, 0, 0, 0, 0, 0, 0, 0, 0, 518, 559, 511, 466, 510, 533, 477, 479, 0, 0, 0, 0, 0, 0, 0, 0, 518, 494, 469, 518, 536, 503, 510, 506, 27, 0, 0, 0, 0, 0, 0, 0, 516, 505, 520, 516, 551, 502, 537, 503, 14, 4, 0, 0, 0, 0, 0, 0, 515, 543, 508, 525, 473, 504, 480, 533, 6, 4, 3, 0, 0, 0, 0, 0, 519, 474, 514, 497, 485, 522, 498, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4053, 4106, 4053, 4081, 4168, 4094, 4038] avg=4096.0000 max=4175 max_rank=0 min=4038 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[508, 517, 495, 504, 479, 524, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 507, 509, 539, 498, 494, 537, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 511, 465, 451, 514, 540, 549, 0, 0, 0, 0, 0, 0, 0, 0, 543, 538, 517, 510, 511, 484, 468, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 524, 500, 506, 490, 527, 502, 506, 27, 0, 0, 0, 0, 0, 0, 0, 492, 506, 493, 540, 520, 513, 506, 545, 14, 4, 0, 0, 0, 0, 0, 0, 494, 508, 520, 504, 532, 510, 475, 505, 6, 4, 4, 0, 0, 0, 0, 0, 508, 506, 524, 505, 505, 536, 531, 556, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4097, 4050, 4098, 4097, 4133, 4062, 4171] avg=4096.0000 max=4171 max_rank=7 min=4050 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 513, 478, 564, 511, 500, 509, 555, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 538, 498, 488, 532, 530, 524, 0, 0, 0, 0, 0, 0, 0, 0, 554, 525, 525, 514, 506, 515, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 512, 479, 531, 527, 492, 540, 489, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 493, 485, 522, 508, 482, 483, 502, 28, 0, 0, 0, 0, 0, 0, 0, 532, 507, 543, 477, 519, 496, 473, 498, 15, 4, 0, 0, 0, 0, 0, 0, 508, 488, 498, 543, 513, 517, 536, 511, 6, 5, 3, 0, 0, 0, 0, 0, 545, 520, 483, 487, 501, 491, 508, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4135, 4176, 4066, 4001, 4064, 4128, 4047] avg=4096.0000 max=4176 max_rank=2 min=4001 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[510, 450, 510, 485, 510, 505, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 538, 544, 543, 510, 523, 494, 490, 519, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 517, 517, 491, 514, 502, 540, 0, 0, 0, 0, 0, 0, 0, 0, 499, 518, 505, 541, 522, 498, 490, 522, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 551, 496, 491, 491, 533, 490, 28, 0, 0, 0, 0, 0, 0, 0, 501, 507, 524, 520, 505, 488, 516, 500, 14, 4, 0, 0, 0, 0, 0, 0, 494, 535, 512, 494, 564, 484, 541, 501, 6, 4, 4, 0, 0, 0, 0, 0, 507, 538, 520, 530, 537, 524, 506, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4161, 4083, 4095, 4080, 4079, 4139, 4137] avg=4096.0000 max=4161 max_rank=1 min=3994 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[466, 517, 508, 479, 501, 497, 520, 500, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 499, 503, 501, 498, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 526, 502, 483, 486, 516, 497, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 513, 528, 541, 512, 520, 512, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0, 450, 535, 555, 503, 538, 514, 516, 528, 34, 0, 0, 0, 0, 0, 0, 0, 510, 509, 460, 561, 487, 465, 510, 518, 19, 6, 6, 0, 0, 0, 0, 0, 494, 536, 533, 496, 538, 512, 552, 496, 8, 10, 7, 0, 0, 0, 0, 0, 506, 528, 538, 501, 538, 516, 495, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 4071, 4024, 4137, 4173, 4051, 4182, 4142] avg=4096.0000 max=4182 max_rank=6 min=3988 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[520, 462, 482, 502, 546, 505, 537, 531, 533, 489, 495, 490, 498, 515, 541, 534, 517, 511, 506, 510, 535, 522, 533, 523, 496, 550, 500, 481, 497, 499, 496, 462, 512, 506, 531, 546, 522, 520, 542, 503, 514, 487, 520, 471, 512, 507, 532, 479, 526, 517, 515, 466, 506, 539, 524, 532, 533, 483, 479, 538, 518, 518, 530, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[537, 518, 539, 526, 499, 522, 499, 502, 492, 512, 493, 491, 524, 479, 529, 501, 521, 497, 525, 528, 531, 501, 494, 450, 498, 479, 483, 509, 485, 519, 505, 484, 529, 533, 556, 500, 533, 494, 585, 537, 499, 520, 515, 519, 504, 514, 524, 507, 511, 508, 531, 490, 521, 532, 528, 513, 520, 491, 524, 519, 518, 501, 528, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[485, 523, 456, 565, 531, 522, 532, 511, 488, 536, 516, 504, 505, 498, 489, 499, 517, 529, 510, 543, 543, 517, 520, 494, 523, 539, 511, 527, 507, 520, 534, 507, 499, 504, 511, 517, 503, 517, 487, 536, 501, 488, 510, 503, 522, 498, 504, 522, 522, 474, 495, 508, 498, 536, 504, 498, 504, 493, 519, 518, 488, 517, 549, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[497, 515, 548, 566, 486, 491, 502, 521, 478, 515, 491, 519, 522, 498, 524, 552, 514, 536, 506, 534, 525, 466, 528, 536, 537, 529, 504, 505, 522, 539, 503, 485, 506, 501, 511, 536, 492, 459, 526, 491, 534, 511, 485, 500, 532, 539, 514, 510, 464, 505, 525, 497, 511, 491, 532, 536, 516, 512, 500, 512, 505, 485, 515, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[486, 468, 533, 514, 522, 523, 490, 504, 495, 506, 544, 505, 512, 490, 489, 492, 549, 533, 476, 494, 522, 514, 511, 489, 538, 538, 529, 499, 485, 472, 520, 521, 494, 530, 537, 488, 500, 507, 494, 525, 512, 578, 507, 473, 531, 530, 496, 507, 512, 533, 533, 539, 487, 498, 493, 524, 481, 530, 529, 525, 533, 519, 541, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[494, 507, 514, 539, 499, 523, 538, 536, 502, 537, 498, 472, 507, 518, 525, 504, 510, 487, 507, 523, 524, 514, 525, 494, 507, 533, 499, 510, 520, 485, 495, 504, 512, 506, 520, 565, 484, 511, 520, 500, 499, 480, 522, 525, 525, 508, 533, 529, 512, 534, 551, 502, 518, 485, 551, 468, 521, 462, 522, 513, 531, 492, 515, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[521, 543, 489, 544, 479, 493, 514, 501, 509, 525, 523, 521, 507, 529, 547, 553, 487, 504, 521, 497, 525, 526, 472, 499, 497, 517, 546, 502, 480, 481, 529, 514, 516, 505, 526, 531, 487, 554, 538, 467, 491, 505, 509, 470, 486, 521, 549, 531, 510, 503, 534, 518, 504, 511, 497, 465, 557, 483, 506, 487, 553, 508, 549, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[527, 500, 507, 512, 507, 512, 503, 472, 507, 489, 494, 540, 518, 518, 572, 525, 514, 533, 500, 514, 471, 531, 507, 523, 521, 535, 510, 510, 526, 512, 541, 511, 505, 485, 491, 530, 505, 461, 488, 548, 499, 505, 518, 526, 495, 513, 500, 514, 517, 551, 533, 525, 510, 493, 520, 488, 510, 515, 546, 495, 492, 507, 518, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[497, 515, 548, 563, 486, 491, 502, 521, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 491, 519, 522, 498, 524, 552, 3, 1, 0, 0, 0, 0, 0, 0, 514, 536, 506, 534, 520, 466, 528, 536, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 504, 505, 522, 539, 503, 485, 13, 4, 0, 0, 0, 0, 0, 0, 506, 501, 511, 523, 492, 459, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 485, 500, 532, 539, 514, 510, 5, 4, 0, 0, 0, 0, 0, 0, 464, 505, 525, 497, 511, 491, 532, 536, 0, 0, 0, 0, 0, 0, 0, 0, 516, 512, 500, 512, 505, 485, 506, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4103, 4140, 4141, 4009, 4134, 4061, 4057] avg=4096.0000 max=4141 max_rank=3 min=4009 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[537, 518, 539, 524, 499, 522, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 512, 493, 491, 524, 479, 529, 501, 2, 1, 0, 0, 0, 0, 0, 0, 521, 497, 525, 528, 526, 501, 494, 450, 0, 0, 0, 0, 0, 0, 0, 0, 498, 479, 483, 509, 485, 519, 505, 484, 12, 4, 0, 0, 0, 0, 0, 0, 529, 533, 556, 488, 533, 494, 585, 537, 0, 0, 0, 0, 0, 0, 0, 0, 499, 520, 515, 519, 504, 514, 524, 507, 6, 4, 0, 0, 0, 0, 0, 0, 511, 508, 531, 490, 521, 532, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 491, 524, 519, 518, 501, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4024, 4042, 3978, 4255, 4112, 4134, 4083] avg=4096.0000 max=4255 max_rank=4 min=3978 min_rank=3 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 468, 533, 512, 522, 523, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 544, 505, 512, 490, 489, 492, 2, 1, 0, 0, 0, 0, 0, 0, 549, 533, 476, 494, 517, 514, 511, 489, 0, 0, 0, 0, 0, 0, 0, 0, 538, 538, 529, 499, 485, 472, 520, 521, 12, 4, 0, 0, 0, 0, 0, 0, 494, 530, 537, 476, 500, 507, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 578, 507, 473, 531, 530, 496, 507, 6, 4, 0, 0, 0, 0, 0, 0, 512, 533, 533, 539, 487, 498, 493, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 530, 529, 525, 533, 519, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4036, 4083, 4118, 4063, 4144, 4119, 4167] avg=4096.0000 max=4167 max_rank=7 min=4036 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 462, 482, 500, 546, 505, 537, 531, 0, 0, 0, 0, 0, 0, 0, 0, 533, 489, 495, 490, 498, 515, 541, 534, 2, 1, 0, 0, 0, 0, 0, 0, 517, 511, 506, 510, 530, 522, 533, 523, 0, 0, 0, 0, 0, 0, 0, 0, 496, 550, 500, 481, 497, 499, 496, 462, 13, 4, 0, 0, 0, 0, 0, 0, 512, 506, 531, 533, 522, 520, 542, 503, 0, 0, 0, 0, 0, 0, 0, 0, 514, 487, 520, 471, 512, 507, 532, 479, 6, 4, 0, 0, 0, 0, 0, 0, 526, 517, 515, 466, 506, 539, 524, 532, 0, 0, 0, 0, 0, 0, 0, 0, 533, 483, 479, 538, 518, 518, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4098, 4152, 3998, 4169, 4032, 4125, 4111] avg=4096.0000 max=4169 max_rank=4 min=3998 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[485, 523, 456, 562, 531, 522, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 488, 536, 516, 504, 505, 498, 489, 499, 3, 1, 0, 0, 0, 0, 0, 0, 517, 529, 510, 543, 538, 517, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 523, 539, 511, 527, 507, 520, 534, 507, 13, 5, 0, 0, 0, 0, 0, 0, 499, 504, 511, 504, 503, 517, 487, 536, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 510, 503, 522, 498, 504, 522, 6, 4, 0, 0, 0, 0, 0, 0, 522, 474, 495, 508, 498, 536, 504, 498, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 519, 518, 488, 517, 538, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4039, 4168, 4186, 4061, 4058, 4035, 4099] avg=4096.0000 max=4186 max_rank=3 min=4035 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[494, 507, 514, 536, 499, 523, 538, 536, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 498, 472, 507, 518, 525, 504, 3, 1, 0, 0, 0, 0, 0, 0, 510, 487, 507, 523, 519, 514, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 499, 510, 520, 485, 495, 504, 14, 4, 0, 0, 0, 0, 0, 0, 512, 506, 520, 551, 484, 511, 520, 500, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 522, 525, 525, 508, 533, 529, 5, 4, 0, 0, 0, 0, 0, 0, 512, 534, 551, 502, 518, 485, 551, 468, 0, 0, 0, 0, 0, 0, 0, 0, 521, 462, 522, 513, 531, 492, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4067, 4079, 4071, 4104, 4130, 4121, 4049] avg=4096.0000 max=4147 max_rank=0 min=4049 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[521, 543, 489, 541, 479, 493, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 523, 521, 507, 529, 547, 553, 3, 1, 0, 0, 0, 0, 0, 0, 487, 504, 521, 497, 520, 526, 472, 499, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 546, 502, 480, 481, 529, 514, 13, 5, 0, 0, 0, 0, 0, 0, 516, 505, 526, 518, 487, 554, 538, 467, 0, 0, 0, 0, 0, 0, 0, 0, 491, 505, 509, 470, 486, 521, 549, 531, 6, 4, 0, 0, 0, 0, 0, 0, 510, 503, 534, 518, 504, 511, 497, 465, 0, 0, 0, 0, 0, 0, 0, 0, 557, 483, 506, 487, 553, 508, 538, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4218, 4026, 4084, 4111, 4072, 4042, 4134] avg=4096.0000 max=4218 max_rank=1 min=4026 min_rank=2 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 500, 507, 506, 507, 512, 503, 472, 0, 0, 0, 0, 0, 0, 0, 0, 507, 489, 494, 540, 518, 518, 572, 525, 6, 8, 6, 0, 0, 0, 0, 0, 514, 533, 500, 514, 456, 531, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 510, 510, 526, 512, 541, 511, 17, 9, 0, 0, 0, 0, 0, 0, 505, 485, 491, 513, 505, 461, 488, 548, 0, 0, 0, 0, 0, 0, 0, 0, 499, 505, 518, 526, 495, 513, 500, 514, 9, 7, 0, 0, 0, 0, 0, 0, 517, 551, 527, 525, 510, 493, 520, 488, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 546, 495, 492, 507, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4183, 4078, 4192, 3996, 4086, 4131, 4068] avg=4096.0000 max=4192 max_rank=3 min=3996 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4352, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True + [2026-08-26 15:43:12] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 274.0 | throughput per GPU (TFLOP/s/GPU): 450.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.031195E+01 | loss scale: 1.0 | grad norm: 8.105 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[519, 527, 488, 532, 500, 490, 512, 507, 487, 511, 526, 491, 533, 522, 530, 504, 556, 545, 501, 547, 549, 497, 510, 524, 505, 487, 491, 480, 511, 522, 510, 512, 526, 541, 526, 491, 520, 528, 541, 538, 509, 493, 509, 496, 528, 530, 512, 520, 511, 484, 490, 497, 522, 539, 489, 500, 489, 486, 520, 482, 507, 498, 509, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[492, 523, 515, 518, 541, 495, 533, 532, 502, 545, 539, 513, 515, 514, 507, 493, 513, 535, 520, 484, 501, 492, 525, 530, 485, 509, 536, 520, 477, 564, 516, 492, 468, 542, 513, 495, 529, 504, 524, 504, 527, 507, 492, 519, 494, 485, 496, 486, 542, 518, 528, 497, 483, 553, 530, 526, 518, 495, 497, 503, 519, 493, 482, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[539, 539, 469, 500, 506, 499, 527, 497, 477, 509, 491, 492, 525, 559, 521, 503, 507, 486, 546, 516, 555, 476, 462, 509, 525, 521, 521, 516, 509, 474, 491, 484, 466, 513, 511, 486, 517, 510, 507, 543, 497, 525, 479, 533, 533, 547, 524, 526, 493, 483, 531, 534, 541, 489, 559, 522, 519, 531, 493, 498, 538, 523, 526, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[514, 518, 508, 538, 519, 505, 505, 492, 518, 538, 515, 505, 506, 484, 518, 526, 515, 540, 489, 504, 497, 549, 506, 538, 490, 513, 507, 498, 488, 489, 542, 480, 543, 561, 528, 507, 511, 510, 515, 514, 503, 513, 518, 502, 512, 503, 488, 520, 496, 521, 502, 549, 497, 532, 512, 514, 532, 470, 509, 492, 501, 531, 487, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[505, 491, 548, 510, 489, 518, 519, 505, 515, 504, 499, 528, 505, 506, 506, 496, 492, 536, 506, 541, 493, 502, 499, 513, 544, 480, 506, 527, 483, 530, 483, 512, 500, 528, 527, 512, 524, 497, 510, 516, 510, 481, 547, 497, 514, 517, 504, 523, 529, 517, 497, 486, 576, 508, 526, 513, 480, 531, 563, 480, 538, 484, 531, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[570, 542, 497, 498, 547, 493, 532, 530, 552, 488, 507, 511, 508, 486, 506, 532, 541, 477, 514, 550, 520, 572, 500, 499, 543, 492, 496, 477, 509, 503, 537, 549, 480, 532, 505, 498, 499, 487, 481, 518, 506, 513, 505, 498, 477, 540, 491, 493, 543, 521, 482, 528, 492, 520, 480, 494, 530, 543, 523, 561, 508, 457, 499, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[499, 497, 505, 502, 501, 529, 499, 528, 504, 484, 498, 529, 491, 501, 495, 541, 483, 511, 561, 529, 491, 478, 488, 552, 531, 507, 509, 522, 510, 563, 508, 480, 519, 508, 480, 547, 484, 472, 464, 550, 533, 489, 520, 547, 479, 503, 521, 529, 566, 527, 478, 490, 547, 529, 525, 479, 516, 554, 522, 486, 517, 508, 530, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[511, 524, 468, 512, 488, 495, 514, 447, 522, 504, 508, 507, 542, 530, 549, 531, 531, 519, 524, 540, 520, 535, 483, 486, 506, 532, 514, 510, 495, 540, 522, 537, 527, 487, 526, 496, 515, 483, 487, 521, 475, 508, 488, 491, 520, 521, 536, 507, 527, 509, 531, 506, 495, 496, 517, 486, 535, 511, 507, 565, 508, 498, 509, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[519, 527, 488, 532, 500, 490, 512, 507, 6, 0, 0, 0, 0, 0, 0, 0, 487, 511, 526, 491, 533, 522, 526, 504, 0, 0, 0, 0, 0, 0, 0, 0, 556, 545, 501, 517, 549, 497, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 491, 480, 511, 522, 510, 512, 10, 7, 0, 0, 0, 0, 0, 0, 526, 541, 526, 491, 520, 528, 541, 538, 4, 2, 0, 0, 0, 0, 0, 0, 509, 493, 509, 496, 528, 530, 512, 520, 20, 0, 0, 0, 0, 0, 0, 0, 496, 484, 490, 497, 522, 539, 489, 500, 0, 0, 0, 0, 0, 0, 0, 0, 489, 486, 520, 482, 507, 498, 509, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4100, 4199, 4035, 4217, 4117, 4017, 4002] avg=4096.0000 max=4217 max_rank=4 min=4002 min_rank=7 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 523, 515, 518, 541, 495, 533, 532, 7, 0, 0, 0, 0, 0, 0, 0, 502, 545, 539, 513, 515, 514, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 513, 535, 520, 457, 501, 492, 525, 530, 0, 0, 0, 0, 0, 0, 0, 0, 485, 509, 536, 520, 477, 564, 516, 492, 9, 8, 0, 0, 0, 0, 0, 0, 468, 542, 513, 495, 529, 504, 524, 504, 4, 2, 0, 0, 0, 0, 0, 0, 527, 507, 492, 519, 494, 485, 496, 486, 18, 0, 0, 0, 0, 0, 0, 0, 525, 518, 528, 497, 483, 553, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 497, 503, 519, 493, 482, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4124, 4073, 4116, 4085, 4024, 4160, 4030] avg=4096.0000 max=4160 max_rank=6 min=4024 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[539, 539, 469, 500, 506, 499, 527, 497, 6, 0, 0, 0, 0, 0, 0, 0, 477, 509, 491, 492, 525, 559, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 507, 486, 546, 487, 555, 476, 462, 509, 0, 0, 0, 0, 0, 0, 0, 0, 525, 521, 521, 516, 509, 474, 491, 484, 10, 7, 0, 0, 0, 0, 0, 0, 466, 513, 511, 486, 517, 510, 507, 543, 4, 1, 0, 0, 0, 0, 0, 0, 497, 525, 479, 533, 533, 547, 524, 526, 19, 0, 0, 0, 0, 0, 0, 0, 479, 483, 531, 534, 541, 489, 559, 522, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 493, 498, 538, 523, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4073, 4028, 4058, 4058, 4183, 4138, 4148] avg=4096.0000 max=4183 max_rank=5 min=4028 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 518, 508, 538, 519, 505, 505, 492, 6, 0, 0, 0, 0, 0, 0, 0, 518, 538, 515, 505, 506, 484, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 515, 540, 489, 476, 497, 549, 506, 538, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 507, 498, 488, 489, 542, 480, 9, 7, 0, 0, 0, 0, 0, 0, 543, 561, 528, 507, 511, 510, 515, 514, 4, 2, 0, 0, 0, 0, 0, 0, 503, 513, 518, 502, 512, 503, 488, 520, 19, 0, 0, 0, 0, 0, 0, 0, 481, 521, 502, 549, 497, 532, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 532, 470, 509, 492, 501, 531, 487, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4106, 4110, 4023, 4195, 4078, 4108, 4043] avg=4096.0000 max=4195 max_rank=4 min=4023 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[511, 524, 468, 512, 488, 495, 514, 447, 7, 0, 0, 0, 0, 0, 0, 0, 522, 504, 508, 507, 542, 530, 545, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 524, 510, 520, 535, 483, 486, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 514, 510, 495, 540, 522, 537, 10, 7, 0, 0, 0, 0, 0, 0, 527, 487, 526, 496, 515, 483, 487, 521, 4, 2, 0, 0, 0, 0, 0, 0, 475, 508, 488, 491, 520, 521, 536, 507, 20, 0, 0, 0, 0, 0, 0, 0, 511, 509, 531, 506, 495, 496, 517, 486, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 507, 565, 508, 498, 509, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4189, 4108, 4173, 4048, 4066, 4051, 4167] avg=4096.0000 max=4189 max_rank=1 min=3966 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 491, 548, 510, 489, 518, 519, 505, 7, 0, 0, 0, 0, 0, 0, 0, 515, 504, 499, 528, 505, 506, 502, 496, 0, 0, 0, 0, 0, 0, 0, 0, 492, 536, 506, 511, 493, 502, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 544, 480, 506, 527, 483, 530, 483, 512, 10, 7, 0, 0, 0, 0, 0, 0, 500, 528, 527, 512, 524, 497, 510, 516, 4, 2, 0, 0, 0, 0, 0, 0, 510, 481, 547, 497, 514, 517, 504, 523, 20, 0, 0, 0, 0, 0, 0, 0, 513, 517, 497, 486, 576, 508, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 563, 480, 538, 484, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4055, 4052, 4082, 4120, 4113, 4136, 4118] avg=4096.0000 max=4136 max_rank=6 min=4052 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 497, 505, 502, 501, 529, 499, 528, 7, 0, 0, 0, 0, 0, 0, 0, 504, 484, 498, 529, 491, 501, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 483, 511, 561, 500, 491, 478, 488, 552, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 509, 522, 510, 563, 508, 480, 10, 8, 0, 0, 0, 0, 0, 0, 519, 508, 480, 547, 484, 472, 464, 550, 4, 2, 0, 0, 0, 0, 0, 0, 533, 489, 520, 547, 479, 503, 521, 529, 19, 0, 0, 0, 0, 0, 0, 0, 549, 527, 478, 490, 547, 529, 525, 479, 0, 0, 0, 0, 0, 0, 0, 0, 516, 554, 522, 486, 517, 508, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4039, 4064, 4148, 4030, 4140, 4124, 4156] avg=4096.0000 max=4156 max_rank=7 min=4030 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[570, 542, 497, 498, 547, 493, 532, 530, 10, 0, 0, 0, 0, 0, 0, 0, 552, 488, 507, 511, 508, 486, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 541, 477, 514, 511, 520, 572, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 496, 477, 509, 503, 537, 549, 15, 12, 0, 0, 0, 0, 0, 0, 480, 532, 505, 498, 499, 487, 481, 518, 8, 4, 3, 0, 0, 0, 0, 0, 506, 513, 505, 498, 477, 540, 491, 493, 24, 0, 0, 0, 0, 0, 0, 0, 517, 521, 482, 528, 492, 520, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 530, 543, 523, 561, 505, 457, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 4082, 4134, 4133, 4015, 4047, 4034, 4104] avg=4096.0000 max=4219 max_rank=0 min=4015 min_rank=4 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[521, 523, 546, 488, 482, 506, 521, 478, 545, 488, 538, 522, 493, 530, 533, 541, 504, 470, 483, 526, 583, 509, 475, 519, 510, 478, 491, 503, 500, 546, 537, 518, 524, 525, 534, 462, 516, 493, 504, 528, 517, 497, 542, 529, 528, 526, 545, 523, 474, 503, 508, 480, 519, 512, 519, 507, 525, 488, 480, 504, 504, 524, 508, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[523, 493, 534, 532, 513, 495, 506, 520, 484, 545, 511, 502, 546, 483, 469, 491, 535, 533, 494, 490, 553, 525, 506, 535, 494, 516, 488, 544, 527, 547, 479, 479, 482, 530, 505, 534, 493, 541, 515, 511, 525, 518, 516, 503, 542, 527, 507, 514, 502, 503, 495, 486, 507, 484, 529, 538, 524, 527, 466, 518, 523, 496, 522, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[478, 488, 486, 533, 500, 533, 544, 500, 534, 558, 515, 518, 545, 514, 525, 494, 454, 491, 483, 556, 485, 492, 518, 511, 507, 571, 544, 507, 499, 507, 493, 494, 538, 479, 484, 510, 498, 522, 513, 516, 506, 476, 537, 523, 493, 498, 514, 519, 502, 522, 506, 533, 548, 536, 530, 547, 508, 481, 509, 525, 496, 510, 506, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[493, 527, 510, 511, 523, 499, 523, 495, 517, 471, 545, 542, 507, 497, 522, 519, 523, 494, 512, 511, 502, 529, 486, 546, 541, 497, 488, 550, 509, 495, 518, 535, 551, 517, 519, 481, 518, 518, 520, 471, 495, 495, 481, 490, 527, 514, 517, 513, 512, 467, 545, 501, 492, 573, 523, 502, 481, 502, 544, 524, 528, 457, 552, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[485, 550, 491, 527, 479, 504, 500, 530, 494, 507, 490, 505, 524, 504, 503, 525, 493, 534, 519, 491, 525, 513, 511, 552, 496, 538, 510, 494, 543, 550, 510, 489, 518, 485, 515, 479, 493, 493, 508, 499, 473, 521, 531, 469, 499, 488, 514, 504, 528, 529, 510, 562, 535, 550, 493, 520, 536, 530, 551, 481, 490, 518, 537, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[494, 532, 507, 484, 514, 482, 514, 526, 509, 465, 543, 534, 538, 505, 529, 490, 479, 504, 534, 498, 506, 517, 502, 510, 508, 539, 526, 512, 494, 493, 540, 521, 535, 475, 473, 494, 527, 544, 528, 528, 512, 541, 482, 503, 539, 535, 507, 494, 520, 507, 517, 498, 544, 521, 553, 507, 483, 466, 487, 520, 539, 515, 508, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[490, 521, 532, 499, 515, 521, 481, 522, 554, 513, 502, 511, 504, 522, 537, 552, 520, 499, 555, 528, 503, 540, 505, 569, 508, 520, 538, 501, 543, 521, 496, 523, 464, 505, 513, 478, 501, 515, 481, 514, 509, 508, 496, 547, 503, 481, 497, 512, 508, 527, 485, 539, 499, 549, 487, 502, 535, 518, 476, 510, 490, 482, 498, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[514, 522, 531, 530, 514, 505, 512, 546, 487, 484, 513, 489, 560, 494, 529, 480, 505, 499, 493, 534, 523, 530, 524, 533, 523, 510, 530, 530, 522, 514, 483, 498, 535, 514, 517, 484, 512, 523, 480, 458, 495, 512, 536, 478, 530, 521, 518, 506, 498, 518, 499, 519, 555, 484, 493, 516, 514, 519, 529, 508, 496, 506, 519, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 527, 510, 511, 523, 499, 523, 495, 8, 0, 0, 0, 0, 0, 0, 0, 517, 471, 545, 542, 487, 497, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 512, 511, 502, 529, 486, 532, 0, 0, 0, 0, 0, 0, 0, 0, 541, 497, 488, 550, 509, 477, 518, 535, 0, 0, 0, 0, 0, 0, 0, 0, 551, 517, 519, 481, 518, 518, 520, 471, 28, 14, 0, 0, 0, 0, 0, 0, 495, 495, 481, 490, 527, 514, 517, 513, 6, 0, 0, 0, 0, 0, 0, 0, 512, 467, 545, 501, 492, 545, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 481, 502, 544, 524, 528, 457, 552, 501, 18, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4100, 4089, 4115, 4137, 4038, 4087, 4113] avg=4096.0000 max=4137 max_rank=4 min=4038 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 523, 546, 488, 482, 506, 521, 478, 7, 0, 0, 0, 0, 0, 0, 0, 545, 488, 538, 522, 474, 530, 533, 541, 0, 0, 0, 0, 0, 0, 0, 0, 504, 470, 483, 526, 583, 509, 475, 506, 0, 0, 0, 0, 0, 0, 0, 0, 510, 478, 491, 503, 500, 526, 537, 518, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 534, 462, 516, 493, 504, 528, 25, 13, 0, 0, 0, 0, 0, 0, 517, 497, 542, 529, 528, 526, 545, 523, 6, 0, 0, 0, 0, 0, 0, 0, 474, 503, 508, 480, 519, 487, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 525, 488, 480, 504, 504, 524, 508, 513, 20, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4171, 4056, 4063, 4124, 4213, 3997, 4072] avg=4096.0000 max=4213 max_rank=5 min=3997 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[478, 488, 486, 533, 500, 533, 544, 500, 7, 0, 0, 0, 0, 0, 0, 0, 534, 558, 515, 518, 524, 514, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 454, 491, 483, 556, 485, 492, 518, 499, 0, 0, 0, 0, 0, 0, 0, 0, 507, 571, 544, 507, 499, 488, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 538, 479, 484, 510, 498, 522, 513, 516, 26, 15, 0, 0, 0, 0, 0, 0, 506, 476, 537, 523, 493, 498, 514, 519, 5, 0, 0, 0, 0, 0, 0, 0, 502, 522, 506, 533, 548, 510, 530, 547, 0, 0, 0, 0, 0, 0, 0, 0, 508, 481, 509, 525, 496, 510, 506, 506, 19, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4182, 3978, 4103, 4101, 4071, 4198, 4066] avg=4096.0000 max=4198 max_rank=6 min=3978 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 493, 534, 532, 513, 495, 506, 520, 7, 0, 0, 0, 0, 0, 0, 0, 484, 545, 511, 502, 525, 483, 469, 491, 0, 0, 0, 0, 0, 0, 0, 0, 535, 533, 494, 490, 553, 525, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 516, 488, 544, 527, 527, 479, 479, 0, 0, 0, 0, 0, 0, 0, 0, 482, 530, 505, 534, 493, 541, 515, 511, 24, 15, 0, 0, 0, 0, 0, 0, 525, 518, 516, 503, 542, 527, 507, 514, 6, 0, 0, 0, 0, 0, 0, 0, 502, 503, 495, 486, 507, 460, 529, 538, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 466, 518, 523, 496, 522, 493, 20, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4010, 4158, 4054, 4150, 4158, 4020, 4095] avg=4096.0000 max=4158 max_rank=2 min=4010 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[485, 550, 491, 527, 479, 504, 500, 530, 8, 0, 0, 0, 0, 0, 0, 0, 494, 507, 490, 505, 504, 504, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 519, 491, 525, 513, 511, 538, 0, 0, 0, 0, 0, 0, 0, 0, 496, 538, 510, 494, 543, 529, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 518, 485, 515, 479, 493, 493, 508, 499, 27, 14, 0, 0, 0, 0, 0, 0, 473, 521, 531, 469, 499, 488, 514, 504, 6, 0, 0, 0, 0, 0, 0, 0, 528, 529, 510, 562, 535, 523, 493, 520, 0, 0, 0, 0, 0, 0, 0, 0, 536, 530, 551, 481, 490, 518, 537, 523, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4032, 4124, 4109, 4031, 4005, 4200, 4193] avg=4096.0000 max=4200 max_rank=6 min=4005 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 532, 507, 484, 514, 482, 514, 526, 7, 0, 0, 0, 0, 0, 0, 0, 509, 465, 543, 534, 517, 505, 529, 490, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 534, 498, 506, 517, 502, 498, 0, 0, 0, 0, 0, 0, 0, 0, 508, 539, 526, 512, 494, 475, 540, 521, 0, 0, 0, 0, 0, 0, 0, 0, 535, 475, 473, 494, 527, 544, 528, 528, 25, 15, 0, 0, 0, 0, 0, 0, 512, 541, 482, 503, 539, 535, 507, 494, 5, 0, 0, 0, 0, 0, 0, 0, 520, 507, 517, 498, 544, 496, 553, 507, 0, 0, 0, 0, 0, 0, 0, 0, 483, 466, 487, 520, 539, 515, 508, 517, 18, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4092, 4038, 4115, 4144, 4118, 4142, 4059] avg=4096.0000 max=4144 max_rank=4 min=4038 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 521, 532, 499, 515, 521, 481, 522, 8, 0, 0, 0, 0, 0, 0, 0, 554, 513, 502, 511, 484, 522, 537, 552, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 555, 528, 503, 540, 505, 555, 0, 0, 0, 0, 0, 0, 0, 0, 508, 520, 538, 501, 543, 502, 496, 523, 0, 0, 0, 0, 0, 0, 0, 0, 464, 505, 513, 478, 501, 515, 481, 514, 27, 14, 0, 0, 0, 0, 0, 0, 509, 508, 496, 547, 503, 481, 497, 512, 6, 0, 0, 0, 0, 0, 0, 0, 508, 527, 485, 539, 499, 522, 487, 502, 0, 0, 0, 0, 0, 0, 0, 0, 535, 518, 476, 510, 490, 482, 498, 494, 19, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4175, 4205, 4131, 4012, 4059, 4069, 4028] avg=4096.0000 max=4205 max_rank=2 min=4012 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 522, 531, 530, 514, 505, 512, 546, 11, 7, 0, 0, 0, 0, 0, 0, 487, 484, 513, 489, 530, 494, 529, 480, 0, 0, 0, 0, 0, 0, 0, 0, 505, 499, 493, 534, 523, 530, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 530, 530, 522, 482, 483, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 517, 484, 512, 523, 480, 458, 27, 19, 0, 0, 0, 0, 0, 0, 495, 512, 536, 478, 530, 521, 518, 506, 10, 0, 0, 0, 0, 0, 0, 0, 498, 518, 499, 519, 555, 457, 493, 516, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 529, 508, 496, 506, 519, 515, 25, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4006, 4120, 4078, 4069, 4106, 4055, 4142] avg=4096.0000 max=4192 max_rank=0 min=4006 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[468, 524, 551, 516, 492, 513, 538, 532, 531, 509, 521, 524, 517, 541, 525, 534, 495, 516, 534, 495, 471, 498, 521, 472, 545, 497, 520, 495, 507, 481, 517, 545, 513, 495, 513, 507, 506, 571, 519, 493, 484, 519, 515, 516, 534, 500, 506, 555, 513, 505, 521, 490, 553, 481, 496, 523, 488, 523, 460, 530, 478, 503, 511, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[529, 523, 498, 498, 526, 503, 524, 500, 516, 490, 549, 564, 561, 515, 481, 515, 506, 506, 521, 512, 513, 537, 516, 525, 501, 499, 483, 540, 470, 510, 496, 516, 543, 524, 485, 549, 490, 523, 483, 516, 499, 526, 530, 525, 539, 505, 511, 479, 477, 480, 548, 504, 508, 516, 510, 501, 507, 515, 511, 493, 509, 498, 509, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[521, 532, 527, 518, 554, 529, 536, 477, 530, 501, 499, 515, 504, 512, 521, 492, 546, 537, 509, 508, 512, 474, 502, 528, 483, 530, 509, 520, 524, 509, 518, 513, 529, 512, 503, 521, 498, 498, 498, 529, 511, 496, 556, 501, 496, 536, 478, 513, 559, 499, 517, 512, 495, 479, 507, 527, 502, 475, 496, 469, 515, 514, 551, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[514, 493, 513, 528, 491, 514, 457, 506, 516, 514, 498, 525, 536, 464, 541, 511, 507, 515, 502, 491, 514, 537, 556, 525, 542, 513, 543, 496, 456, 513, 505, 517, 490, 503, 465, 492, 494, 528, 509, 510, 520, 471, 522, 506, 519, 527, 512, 529, 486, 541, 507, 506, 512, 545, 554, 477, 504, 518, 521, 528, 497, 540, 506, 576] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[533, 536, 488, 507, 537, 500, 506, 539, 485, 470, 511, 474, 510, 509, 551, 529, 493, 479, 489, 497, 496, 460, 563, 482, 553, 531, 581, 540, 523, 527, 517, 522, 497, 504, 522, 491, 545, 506, 479, 511, 508, 541, 519, 485, 540, 495, 500, 484, 547, 547, 494, 503, 523, 528, 497, 487, 521, 552, 504, 511, 517, 481, 480, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[522, 521, 506, 555, 504, 493, 506, 476, 503, 518, 469, 493, 486, 516, 475, 520, 511, 530, 505, 523, 508, 502, 510, 509, 478, 477, 527, 501, 502, 513, 513, 467, 495, 503, 498, 509, 525, 499, 512, 529, 522, 536, 511, 537, 518, 535, 499, 540, 528, 526, 499, 559, 530, 501, 477, 595, 524, 491, 521, 541, 510, 495, 524, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[457, 526, 521, 486, 503, 483, 511, 520, 522, 475, 510, 501, 485, 500, 537, 491, 492, 515, 494, 487, 500, 508, 481, 551, 557, 510, 500, 511, 535, 530, 546, 513, 541, 536, 489, 526, 486, 524, 507, 484, 512, 536, 500, 518, 547, 501, 534, 500, 511, 459, 502, 514, 499, 517, 539, 529, 515, 536, 538, 545, 488, 546, 495, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[533, 511, 515, 490, 510, 528, 508, 535, 523, 526, 524, 506, 512, 514, 519, 490, 551, 498, 487, 491, 482, 525, 513, 540, 491, 503, 491, 503, 501, 505, 482, 518, 500, 538, 516, 526, 485, 489, 491, 512, 484, 514, 516, 506, 515, 516, 516, 502, 530, 528, 495, 526, 530, 484, 504, 532, 497, 519, 510, 519, 547, 510, 538, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[468, 520, 551, 516, 492, 513, 538, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 521, 524, 517, 541, 525, 534, 3, 1, 0, 0, 0, 0, 0, 0, 495, 516, 534, 495, 471, 498, 521, 472, 19, 5, 0, 0, 0, 0, 0, 0, 545, 497, 519, 495, 507, 481, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 513, 507, 506, 571, 519, 493, 10, 10, 1, 0, 0, 0, 0, 0, 484, 519, 515, 516, 515, 500, 506, 555, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 521, 490, 553, 481, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 488, 523, 460, 530, 478, 503, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4206, 4026, 4106, 4138, 4110, 4067, 3985] avg=4096.0000 max=4206 max_rank=1 min=3985 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 528, 527, 518, 554, 529, 536, 477, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 499, 515, 504, 512, 521, 492, 3, 1, 0, 0, 0, 0, 0, 0, 546, 537, 509, 508, 512, 474, 502, 528, 18, 5, 0, 0, 0, 0, 0, 0, 483, 530, 508, 520, 524, 509, 518, 513, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 503, 521, 498, 498, 498, 529, 10, 10, 1, 0, 0, 0, 0, 0, 511, 496, 556, 501, 478, 536, 478, 513, 0, 0, 0, 0, 0, 0, 0, 0, 559, 499, 517, 512, 495, 479, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 475, 496, 469, 515, 514, 551, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4078, 4139, 4105, 4109, 4069, 4080, 3998] avg=4096.0000 max=4190 max_rank=0 min=3998 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 519, 498, 498, 526, 503, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 516, 490, 549, 564, 561, 515, 481, 515, 3, 1, 0, 0, 0, 0, 0, 0, 506, 506, 521, 512, 513, 537, 516, 525, 19, 4, 0, 0, 0, 0, 0, 0, 501, 499, 482, 540, 470, 510, 496, 516, 0, 0, 0, 0, 0, 0, 0, 0, 543, 524, 485, 549, 490, 523, 483, 516, 10, 9, 1, 0, 0, 0, 0, 0, 499, 526, 530, 525, 520, 505, 511, 479, 0, 0, 0, 0, 0, 0, 0, 0, 477, 480, 548, 504, 508, 516, 510, 488, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 511, 493, 509, 498, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4195, 4159, 4014, 4133, 4095, 4031, 4044] avg=4096.0000 max=4195 max_rank=1 min=4014 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[533, 532, 488, 507, 537, 500, 506, 539, 0, 0, 0, 0, 0, 0, 0, 0, 485, 470, 511, 474, 510, 509, 551, 529, 3, 1, 0, 0, 0, 0, 0, 0, 493, 479, 489, 497, 496, 460, 563, 482, 19, 4, 0, 0, 0, 0, 0, 0, 553, 531, 580, 540, 523, 527, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 522, 491, 545, 506, 479, 511, 10, 9, 1, 0, 0, 0, 0, 0, 508, 541, 519, 485, 521, 495, 500, 484, 0, 0, 0, 0, 0, 0, 0, 0, 547, 547, 494, 503, 523, 528, 497, 474, 0, 0, 0, 0, 0, 0, 0, 0, 521, 552, 504, 511, 517, 481, 480, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4043, 3982, 4293, 4075, 4053, 4113, 4067] avg=4096.0000 max=4293 max_rank=3 min=3982 min_rank=2 imbalance_ratio=1.048096 max_load_reduction=28475 max_load_reduction_pct=0.868988 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 489, 513, 528, 491, 514, 457, 506, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 498, 525, 536, 464, 541, 511, 3, 1, 0, 0, 0, 0, 0, 0, 507, 515, 502, 491, 514, 537, 556, 525, 19, 4, 0, 0, 0, 0, 0, 0, 542, 513, 542, 496, 456, 513, 505, 517, 0, 0, 0, 0, 0, 0, 0, 0, 490, 503, 465, 492, 494, 528, 509, 510, 12, 9, 1, 0, 0, 0, 0, 0, 520, 471, 522, 506, 500, 527, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 486, 541, 507, 506, 512, 545, 554, 464, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 521, 528, 497, 540, 506, 564, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4109, 4170, 4084, 4013, 4087, 4115, 4178] avg=4096.0000 max=4178 max_rank=7 min=4012 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[533, 507, 515, 490, 510, 528, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 524, 506, 512, 514, 519, 490, 3, 1, 0, 0, 0, 0, 0, 0, 551, 498, 487, 491, 482, 525, 513, 540, 18, 5, 0, 0, 0, 0, 0, 0, 491, 503, 490, 503, 501, 505, 482, 518, 0, 0, 0, 0, 0, 0, 0, 0, 500, 538, 516, 526, 485, 489, 491, 512, 11, 10, 1, 0, 0, 0, 0, 0, 484, 514, 516, 506, 497, 516, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 495, 526, 530, 484, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 497, 519, 510, 519, 547, 510, 538, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4118, 4110, 3993, 4079, 4051, 4114, 4177] avg=4096.0000 max=4177 max_rank=7 min=3993 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[522, 517, 506, 555, 504, 493, 506, 476, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 469, 493, 486, 516, 475, 520, 3, 1, 0, 0, 0, 0, 0, 0, 511, 530, 505, 523, 508, 502, 510, 509, 19, 5, 0, 0, 0, 0, 0, 0, 478, 477, 526, 501, 502, 513, 513, 467, 0, 0, 0, 0, 0, 0, 0, 0, 495, 503, 498, 509, 525, 499, 512, 529, 11, 11, 1, 0, 0, 0, 0, 0, 522, 536, 511, 537, 499, 535, 499, 540, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 499, 559, 530, 501, 477, 579, 0, 0, 0, 0, 0, 0, 0, 0, 524, 491, 521, 541, 510, 495, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 3984, 4122, 3977, 4093, 4179, 4199, 4135] avg=4096.0000 max=4199 max_rank=6 min=3977 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[457, 511, 521, 486, 503, 483, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 475, 510, 501, 485, 500, 537, 491, 8, 6, 0, 0, 0, 0, 0, 0, 492, 515, 494, 487, 500, 508, 481, 551, 24, 8, 0, 0, 0, 0, 0, 0, 557, 510, 494, 511, 535, 530, 546, 513, 0, 0, 0, 0, 0, 0, 0, 0, 541, 536, 489, 526, 486, 524, 507, 484, 15, 13, 7, 0, 0, 0, 0, 0, 512, 536, 500, 518, 523, 501, 534, 500, 0, 0, 0, 0, 0, 0, 0, 0, 511, 459, 502, 514, 499, 517, 539, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 538, 545, 488, 546, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 4035, 4060, 4196, 4128, 4124, 4049, 4184] avg=4096.0000 max=4196 max_rank=3 min=3992 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[526, 529, 494, 531, 502, 496, 501, 480, 535, 500, 530, 501, 489, 523, 477, 498, 498, 499, 536, 541, 501, 473, 525, 494, 511, 547, 479, 524, 521, 515, 479, 481, 524, 532, 551, 508, 517, 525, 498, 553, 491, 542, 520, 517, 490, 514, 530, 504, 556, 506, 516, 487, 503, 486, 512, 510, 524, 489, 544, 494, 525, 510, 521, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[553, 511, 466, 545, 491, 510, 553, 488, 529, 492, 511, 493, 505, 475, 536, 494, 493, 521, 502, 526, 543, 491, 489, 537, 481, 506, 512, 526, 520, 521, 536, 504, 527, 563, 481, 512, 505, 469, 508, 526, 542, 495, 548, 535, 490, 529, 519, 523, 520, 522, 502, 477, 524, 515, 519, 526, 536, 462, 530, 517, 498, 498, 489, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[452, 515, 548, 534, 484, 528, 521, 500, 526, 526, 519, 507, 546, 515, 478, 490, 526, 544, 496, 509, 511, 520, 503, 516, 479, 510, 553, 472, 537, 517, 480, 504, 528, 506, 539, 500, 532, 488, 471, 506, 522, 509, 477, 532, 525, 535, 481, 541, 487, 493, 503, 484, 475, 532, 502, 511, 523, 498, 508, 544, 553, 538, 539, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[504, 553, 495, 551, 518, 522, 499, 525, 518, 463, 498, 502, 518, 508, 524, 501, 501, 525, 491, 530, 558, 517, 516, 465, 502, 538, 473, 494, 547, 541, 552, 519, 506, 493, 498, 561, 508, 509, 520, 529, 495, 501, 505, 534, 523, 492, 517, 528, 511, 514, 481, 500, 515, 515, 534, 521, 512, 491, 499, 468, 491, 493, 520, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[493, 492, 500, 511, 536, 487, 479, 480, 510, 511, 476, 522, 514, 494, 512, 475, 529, 493, 563, 499, 511, 497, 508, 519, 537, 529, 522, 498, 528, 534, 558, 545, 487, 509, 485, 514, 520, 534, 536, 524, 497, 541, 493, 512, 483, 546, 537, 546, 552, 507, 540, 481, 522, 487, 525, 494, 510, 530, 504, 508, 496, 513, 486, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[497, 530, 495, 521, 512, 532, 525, 541, 493, 510, 498, 479, 495, 504, 493, 483, 552, 506, 541, 489, 505, 473, 480, 560, 454, 540, 502, 498, 516, 522, 534, 515, 520, 515, 502, 525, 517, 489, 517, 494, 527, 483, 512, 496, 525, 530, 531, 529, 521, 516, 516, 515, 546, 521, 485, 491, 523, 507, 511, 563, 511, 507, 490, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[525, 500, 530, 528, 530, 518, 505, 525, 532, 519, 518, 502, 531, 485, 546, 503, 502, 528, 521, 505, 485, 484, 511, 495, 491, 496, 500, 499, 502, 505, 526, 526, 504, 507, 543, 451, 546, 481, 504, 524, 526, 515, 534, 504, 514, 503, 474, 509, 525, 482, 470, 520, 508, 552, 534, 531, 526, 515, 532, 502, 503, 490, 524, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[531, 510, 559, 538, 487, 509, 508, 514, 504, 490, 499, 500, 506, 512, 490, 505, 502, 512, 537, 528, 499, 520, 524, 536, 503, 497, 505, 491, 475, 517, 496, 477, 521, 515, 505, 532, 525, 526, 517, 553, 539, 529, 481, 521, 523, 492, 488, 511, 521, 495, 540, 514, 539, 530, 486, 524, 511, 514, 540, 482, 522, 515, 483, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 529, 494, 519, 502, 496, 501, 480, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 530, 501, 489, 523, 477, 498, 22, 21, 9, 0, 0, 0, 0, 0, 498, 499, 532, 541, 501, 473, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 479, 524, 521, 515, 479, 481, 3, 2, 0, 0, 0, 0, 0, 0, 524, 532, 551, 508, 517, 525, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 491, 542, 520, 517, 490, 514, 530, 482, 0, 0, 0, 0, 0, 0, 0, 0, 556, 506, 516, 487, 503, 486, 512, 510, 2, 0, 0, 0, 0, 0, 0, 0, 524, 489, 544, 494, 525, 510, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4105, 4063, 4062, 4187, 4086, 4078, 4140] avg=4096.0000 max=4187 max_rank=4 min=4047 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[553, 511, 466, 533, 491, 510, 553, 488, 0, 0, 0, 0, 0, 0, 0, 0, 529, 492, 511, 493, 505, 475, 536, 494, 23, 20, 9, 0, 0, 0, 0, 0, 493, 521, 498, 526, 543, 491, 489, 537, 0, 0, 0, 0, 0, 0, 0, 0, 481, 506, 512, 526, 520, 521, 536, 504, 3, 2, 0, 0, 0, 0, 0, 0, 527, 563, 481, 512, 505, 469, 508, 506, 0, 0, 0, 0, 0, 0, 0, 0, 542, 495, 548, 535, 490, 529, 519, 500, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 502, 477, 524, 515, 519, 526, 2, 0, 0, 0, 0, 0, 0, 0, 536, 462, 530, 517, 498, 498, 489, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4087, 4098, 4111, 4071, 4158, 4107, 4031] avg=4096.0000 max=4158 max_rank=5 min=4031 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[452, 515, 548, 522, 484, 528, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 519, 507, 546, 515, 478, 490, 24, 20, 9, 0, 0, 0, 0, 0, 526, 544, 492, 509, 511, 520, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0, 479, 510, 553, 472, 537, 517, 480, 504, 3, 2, 0, 0, 0, 0, 0, 0, 528, 506, 539, 500, 532, 488, 471, 486, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 477, 532, 525, 535, 481, 517, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 503, 484, 475, 532, 502, 511, 2, 0, 0, 0, 0, 0, 0, 0, 523, 498, 508, 544, 553, 538, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4160, 4121, 4057, 4050, 4098, 3989, 4223] avg=4096.0000 max=4223 max_rank=7 min=3989 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 553, 495, 538, 518, 522, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 518, 463, 498, 502, 518, 508, 524, 501, 23, 20, 9, 0, 0, 0, 0, 0, 501, 525, 487, 530, 558, 517, 516, 465, 0, 0, 0, 0, 0, 0, 0, 0, 502, 538, 473, 494, 547, 541, 552, 519, 4, 2, 0, 0, 0, 0, 0, 0, 506, 493, 498, 561, 508, 509, 520, 509, 0, 0, 0, 0, 0, 0, 0, 0, 495, 501, 505, 534, 523, 492, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 481, 500, 515, 515, 534, 521, 2, 0, 0, 0, 0, 0, 0, 0, 512, 491, 499, 468, 491, 493, 520, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4084, 4099, 4172, 4104, 4072, 4093, 3990] avg=4096.0000 max=4172 max_rank=3 min=3990 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 530, 516, 530, 518, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 532, 519, 518, 502, 531, 485, 546, 503, 22, 20, 9, 0, 0, 0, 0, 0, 502, 528, 517, 505, 485, 484, 511, 495, 0, 0, 0, 0, 0, 0, 0, 0, 491, 496, 500, 499, 502, 505, 526, 526, 3, 2, 0, 0, 0, 0, 0, 0, 504, 507, 543, 451, 546, 481, 504, 504, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 534, 504, 514, 503, 474, 487, 0, 0, 0, 0, 0, 0, 0, 0, 525, 482, 470, 520, 508, 552, 534, 531, 2, 0, 0, 0, 0, 0, 0, 0, 526, 515, 532, 502, 503, 490, 524, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4187, 4027, 4050, 4040, 4057, 4124, 4134] avg=4096.0000 max=4187 max_rank=1 min=4027 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[493, 492, 500, 500, 536, 487, 479, 480, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 476, 522, 514, 494, 512, 475, 24, 20, 8, 0, 0, 0, 0, 0, 529, 493, 559, 499, 511, 497, 508, 519, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 522, 498, 528, 534, 558, 545, 3, 2, 0, 0, 0, 0, 0, 0, 487, 509, 485, 514, 520, 534, 536, 504, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 493, 512, 483, 546, 537, 522, 0, 0, 0, 0, 0, 0, 0, 0, 552, 507, 540, 481, 522, 487, 525, 494, 2, 0, 0, 0, 0, 0, 0, 0, 510, 530, 504, 508, 496, 513, 486, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3967, 4066, 4115, 4256, 4089, 4131, 4110, 4034] avg=4096.0000 max=4256 max_rank=3 min=3967 min_rank=0 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[497, 530, 495, 500, 512, 532, 525, 541, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 498, 479, 495, 504, 493, 483, 28, 25, 12, 0, 0, 0, 0, 0, 552, 506, 526, 489, 505, 473, 480, 560, 0, 0, 0, 0, 0, 0, 0, 0, 454, 540, 502, 498, 516, 522, 534, 515, 9, 4, 0, 0, 0, 0, 0, 0, 520, 515, 502, 525, 517, 489, 517, 469, 0, 0, 0, 0, 0, 0, 0, 0, 527, 483, 512, 496, 525, 530, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 516, 515, 546, 521, 485, 491, 5, 0, 0, 0, 0, 0, 0, 0, 523, 507, 511, 563, 511, 507, 490, 538, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4020, 4091, 4094, 4054, 4105, 4116, 4156] avg=4096.0000 max=4156 max_rank=7 min=4020 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[531, 510, 559, 526, 487, 509, 508, 514, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 499, 500, 506, 512, 490, 505, 23, 21, 9, 0, 0, 0, 0, 0, 502, 512, 533, 528, 499, 520, 524, 536, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 505, 491, 475, 517, 496, 477, 3, 2, 0, 0, 0, 0, 0, 0, 521, 515, 505, 532, 525, 526, 517, 532, 0, 0, 0, 0, 0, 0, 0, 0, 539, 529, 481, 521, 523, 492, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 540, 514, 539, 530, 486, 524, 2, 0, 0, 0, 0, 0, 0, 0, 511, 514, 540, 482, 522, 515, 483, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4059, 4154, 3966, 4173, 4061, 4151, 4060] avg=4096.0000 max=4173 max_rank=4 min=3966 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[508, 502, 481, 538, 522, 501, 523, 521, 487, 534, 514, 484, 525, 539, 540, 478, 545, 524, 544, 508, 494, 526, 527, 546, 532, 526, 502, 486, 537, 499, 532, 476, 512, 511, 509, 506, 518, 493, 488, 500, 509, 529, 531, 492, 492, 539, 513, 473, 498, 511, 502, 539, 507, 537, 514, 525, 537, 478, 516, 491, 486, 495, 506, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[525, 517, 503, 530, 524, 517, 480, 503, 469, 522, 487, 491, 511, 479, 562, 528, 543, 476, 502, 485, 508, 502, 492, 536, 489, 533, 546, 507, 520, 515, 522, 486, 493, 531, 496, 507, 522, 540, 475, 516, 508, 497, 516, 464, 546, 528, 516, 536, 487, 518, 518, 521, 511, 526, 513, 481, 488, 518, 540, 584, 500, 493, 543, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[495, 524, 474, 529, 545, 490, 517, 533, 479, 512, 518, 560, 478, 524, 524, 515, 502, 582, 553, 540, 507, 498, 486, 509, 508, 499, 488, 507, 509, 487, 510, 523, 505, 512, 494, 522, 528, 545, 500, 537, 505, 527, 499, 503, 505, 485, 521, 503, 507, 490, 479, 476, 515, 521, 529, 546, 500, 518, 491, 515, 510, 528, 532, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[469, 471, 534, 503, 499, 516, 539, 512, 540, 491, 517, 537, 519, 525, 504, 528, 512, 509, 515, 505, 530, 499, 505, 509, 538, 522, 527, 498, 488, 498, 546, 492, 507, 507, 517, 534, 503, 503, 495, 510, 527, 539, 494, 523, 533, 473, 545, 520, 489, 493, 499, 510, 502, 532, 485, 564, 520, 488, 530, 492, 509, 502, 564, 462] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[492, 524, 510, 490, 502, 526, 505, 524, 488, 473, 521, 537, 487, 503, 480, 463, 517, 508, 542, 509, 524, 514, 526, 561, 546, 500, 474, 498, 517, 538, 496, 535, 544, 550, 503, 522, 481, 537, 499, 516, 483, 509, 479, 497, 531, 522, 512, 530, 474, 491, 497, 491, 527, 541, 501, 529, 571, 488, 500, 523, 536, 521, 505, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[518, 519, 514, 463, 521, 494, 519, 522, 540, 512, 478, 539, 499, 508, 487, 547, 506, 521, 502, 500, 505, 495, 530, 507, 507, 498, 512, 520, 520, 547, 487, 515, 537, 521, 529, 526, 480, 502, 505, 492, 481, 523, 529, 483, 513, 520, 526, 509, 512, 463, 558, 511, 537, 498, 532, 513, 522, 513, 503, 518, 561, 520, 487, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[515, 502, 506, 546, 516, 497, 518, 529, 492, 494, 514, 557, 513, 506, 500, 500, 510, 571, 498, 528, 520, 524, 501, 505, 481, 490, 509, 534, 481, 518, 503, 485, 508, 491, 531, 500, 540, 495, 505, 497, 521, 555, 463, 475, 536, 538, 498, 524, 494, 515, 490, 493, 519, 552, 520, 495, 487, 519, 514, 526, 520, 509, 535, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[485, 497, 515, 541, 491, 448, 532, 556, 521, 530, 531, 478, 495, 508, 513, 508, 562, 555, 472, 530, 497, 505, 530, 523, 488, 493, 519, 471, 512, 522, 511, 517, 532, 503, 509, 510, 515, 481, 537, 532, 501, 504, 534, 541, 499, 514, 515, 477, 462, 533, 493, 543, 500, 518, 525, 525, 486, 500, 491, 567, 492, 545, 524, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 502, 481, 538, 522, 501, 523, 521, 5, 4, 0, 0, 0, 0, 0, 0, 487, 534, 514, 484, 525, 539, 540, 478, 15, 0, 0, 0, 0, 0, 0, 0, 545, 483, 544, 508, 494, 526, 527, 546, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 502, 486, 537, 499, 532, 476, 21, 0, 0, 0, 0, 0, 0, 0, 512, 511, 509, 506, 518, 493, 488, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 531, 492, 492, 539, 513, 473, 4, 0, 0, 0, 0, 0, 0, 0, 498, 511, 502, 539, 507, 537, 514, 525, 8, 0, 0, 0, 0, 0, 0, 0, 537, 478, 516, 475, 486, 495, 506, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4116, 4173, 4111, 4037, 4082, 4141, 4003] avg=4096.0000 max=4173 max_rank=2 min=4003 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[495, 524, 474, 529, 545, 490, 517, 533, 6, 4, 0, 0, 0, 0, 0, 0, 479, 512, 518, 560, 478, 524, 524, 515, 17, 0, 0, 0, 0, 0, 0, 0, 502, 535, 553, 540, 507, 498, 486, 509, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 488, 507, 509, 487, 510, 523, 24, 0, 0, 0, 0, 0, 0, 0, 505, 512, 494, 522, 528, 545, 500, 537, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 499, 503, 505, 485, 521, 503, 4, 0, 0, 0, 0, 0, 0, 0, 507, 490, 479, 476, 515, 521, 529, 546, 8, 0, 0, 0, 0, 0, 0, 0, 500, 518, 491, 499, 510, 528, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4127, 4130, 4055, 4143, 4052, 4071, 4073] avg=4096.0000 max=4143 max_rank=4 min=4052 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 517, 503, 530, 524, 517, 480, 503, 5, 5, 0, 0, 0, 0, 0, 0, 469, 522, 487, 491, 511, 479, 562, 528, 14, 0, 0, 0, 0, 0, 0, 0, 543, 438, 502, 485, 508, 502, 492, 536, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 546, 507, 520, 515, 522, 486, 19, 0, 0, 0, 0, 0, 0, 0, 493, 531, 496, 507, 522, 540, 475, 516, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 516, 464, 546, 528, 516, 536, 4, 0, 0, 0, 0, 0, 0, 0, 487, 518, 518, 521, 511, 526, 513, 481, 9, 0, 0, 0, 0, 0, 0, 0, 488, 518, 540, 566, 500, 493, 543, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4063, 4006, 4137, 4080, 4115, 4084, 4174] avg=4096.0000 max=4174 max_rank=7 min=4006 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[492, 524, 510, 490, 502, 526, 505, 524, 5, 4, 0, 0, 0, 0, 0, 0, 488, 473, 521, 537, 487, 503, 480, 463, 14, 0, 0, 0, 0, 0, 0, 0, 517, 468, 542, 509, 524, 514, 526, 561, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 474, 498, 517, 538, 496, 535, 21, 0, 0, 0, 0, 0, 0, 0, 544, 550, 503, 522, 481, 537, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 479, 497, 531, 522, 512, 530, 4, 0, 0, 0, 0, 0, 0, 0, 474, 491, 497, 491, 527, 541, 501, 529, 8, 0, 0, 0, 0, 0, 0, 0, 571, 488, 500, 507, 536, 521, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 3966, 4161, 4125, 4152, 4067, 4059, 4156] avg=4096.0000 max=4161 max_rank=2 min=3966 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[469, 471, 534, 503, 499, 516, 539, 512, 5, 4, 0, 0, 0, 0, 0, 0, 540, 491, 517, 537, 519, 525, 504, 528, 14, 0, 0, 0, 0, 0, 0, 0, 512, 469, 515, 505, 530, 499, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 527, 498, 488, 498, 546, 492, 21, 0, 0, 0, 0, 0, 0, 0, 507, 507, 517, 534, 503, 503, 495, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 539, 494, 523, 533, 473, 545, 520, 4, 0, 0, 0, 0, 0, 0, 0, 489, 493, 499, 510, 502, 532, 485, 564, 8, 0, 0, 0, 0, 0, 0, 0, 520, 488, 530, 476, 509, 502, 564, 462, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4175, 4044, 4130, 4076, 4158, 4082, 4051] avg=4096.0000 max=4175 max_rank=1 min=4044 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 519, 514, 463, 521, 494, 519, 522, 5, 4, 0, 0, 0, 0, 0, 0, 540, 512, 478, 539, 499, 508, 487, 547, 15, 0, 0, 0, 0, 0, 0, 0, 506, 480, 502, 500, 505, 495, 530, 507, 0, 0, 0, 0, 0, 0, 0, 0, 507, 498, 512, 520, 520, 547, 487, 515, 21, 0, 0, 0, 0, 0, 0, 0, 537, 521, 529, 526, 480, 502, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 529, 483, 513, 520, 526, 509, 4, 0, 0, 0, 0, 0, 0, 0, 512, 463, 558, 511, 537, 498, 532, 513, 8, 0, 0, 0, 0, 0, 0, 0, 522, 513, 503, 502, 561, 520, 487, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4125, 4025, 4127, 4092, 4088, 4132, 4100] avg=4096.0000 max=4132 max_rank=6 min=4025 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[515, 502, 506, 546, 516, 497, 518, 529, 11, 8, 0, 0, 0, 0, 0, 0, 492, 494, 514, 557, 513, 506, 500, 500, 20, 0, 0, 0, 0, 0, 0, 0, 510, 514, 498, 528, 520, 524, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 490, 509, 534, 481, 518, 503, 485, 26, 0, 0, 0, 0, 0, 0, 0, 508, 491, 531, 500, 540, 495, 505, 497, 2, 0, 0, 0, 0, 0, 0, 0, 521, 555, 463, 475, 536, 538, 498, 524, 7, 0, 0, 0, 0, 0, 0, 0, 494, 515, 490, 493, 519, 552, 520, 495, 13, 0, 0, 0, 0, 0, 0, 0, 487, 519, 514, 496, 520, 509, 535, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4096, 4100, 4027, 4069, 4117, 4091, 4120] avg=4096.0000 max=4148 max_rank=0 min=4027 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 497, 515, 541, 491, 448, 532, 556, 6, 5, 0, 0, 0, 0, 0, 0, 521, 530, 531, 478, 495, 508, 513, 508, 16, 0, 0, 0, 0, 0, 0, 0, 562, 510, 472, 530, 497, 505, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0, 488, 493, 519, 471, 512, 522, 511, 517, 23, 0, 0, 0, 0, 0, 0, 0, 532, 503, 509, 510, 515, 481, 537, 532, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 534, 541, 499, 514, 515, 477, 4, 0, 0, 0, 0, 0, 0, 0, 462, 533, 493, 543, 500, 518, 525, 525, 9, 0, 0, 0, 0, 0, 0, 0, 486, 500, 491, 549, 492, 545, 524, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4100, 4129, 4056, 4119, 4089, 4108, 4091] avg=4096.0000 max=4129 max_rank=2 min=4056 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4096, 4224, 4352, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[538, 508, 487, 547, 517, 516, 501, 538, 507, 526, 521, 521, 489, 488, 524, 527, 500, 492, 528, 457, 537, 473, 535, 508, 513, 535, 524, 512, 486, 512, 515, 498, 507, 505, 518, 495, 510, 494, 528, 534, 485, 530, 542, 511, 530, 516, 518, 500, 526, 493, 492, 526, 502, 518, 516, 534, 512, 510, 489, 486, 523, 524, 509, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[526, 532, 538, 489, 526, 505, 510, 519, 509, 527, 540, 492, 515, 491, 552, 497, 534, 464, 534, 494, 528, 497, 486, 535, 510, 510, 490, 515, 510, 514, 529, 545, 512, 487, 489, 476, 528, 530, 485, 498, 499, 458, 551, 479, 522, 503, 486, 510, 499, 512, 534, 517, 550, 519, 496, 506, 502, 532, 580, 500, 482, 484, 549, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[508, 526, 497, 545, 499, 482, 492, 478, 518, 524, 496, 539, 515, 523, 490, 536, 531, 509, 548, 512, 476, 507, 507, 484, 532, 530, 540, 501, 535, 490, 487, 515, 536, 496, 480, 551, 515, 520, 555, 527, 511, 479, 514, 495, 504, 535, 481, 512, 503, 527, 483, 518, 499, 504, 532, 505, 474, 546, 498, 544, 528, 502, 507, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[514, 503, 469, 542, 501, 487, 529, 489, 546, 511, 507, 498, 514, 512, 503, 508, 496, 522, 497, 499, 501, 511, 504, 506, 475, 522, 525, 514, 538, 525, 542, 510, 532, 508, 515, 522, 516, 512, 470, 510, 513, 500, 502, 543, 525, 533, 519, 489, 498, 529, 525, 486, 516, 542, 507, 443, 532, 531, 529, 501, 538, 561, 498, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[485, 518, 515, 522, 514, 509, 477, 512, 521, 526, 506, 539, 508, 518, 524, 501, 545, 492, 519, 509, 515, 496, 470, 503, 467, 526, 479, 482, 562, 498, 519, 540, 516, 504, 502, 505, 516, 512, 506, 491, 531, 532, 531, 517, 566, 500, 485, 548, 545, 495, 496, 528, 494, 476, 512, 504, 525, 479, 522, 520, 498, 534, 518, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[573, 489, 549, 481, 522, 526, 541, 532, 494, 506, 506, 482, 493, 505, 531, 482, 475, 508, 514, 481, 532, 511, 515, 541, 564, 529, 510, 504, 536, 480, 456, 522, 530, 514, 517, 525, 515, 492, 549, 504, 493, 488, 492, 511, 505, 496, 516, 482, 486, 498, 511, 538, 507, 498, 518, 540, 521, 475, 513, 536, 524, 523, 513, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[485, 524, 533, 531, 514, 513, 501, 507, 525, 535, 522, 493, 472, 489, 533, 549, 511, 504, 501, 517, 515, 520, 517, 505, 487, 497, 503, 541, 532, 502, 527, 459, 553, 511, 474, 545, 502, 519, 530, 504, 547, 513, 481, 485, 523, 527, 526, 495, 511, 531, 507, 506, 513, 528, 506, 538, 459, 529, 488, 562, 504, 511, 466, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[511, 527, 515, 529, 514, 509, 499, 507, 496, 538, 511, 511, 532, 529, 510, 487, 493, 520, 515, 515, 483, 483, 516, 506, 485, 514, 513, 582, 440, 507, 528, 525, 531, 545, 435, 522, 508, 518, 539, 515, 531, 459, 512, 510, 525, 482, 491, 528, 521, 485, 521, 498, 507, 509, 548, 513, 469, 541, 549, 522, 518, 532, 530, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[526, 532, 538, 482, 526, 505, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 540, 492, 515, 491, 552, 497, 0, 0, 0, 0, 0, 0, 0, 0, 534, 464, 534, 494, 528, 497, 486, 535, 28, 7, 0, 0, 0, 0, 0, 0, 510, 505, 490, 515, 510, 514, 529, 545, 0, 0, 0, 0, 0, 0, 0, 0, 508, 487, 489, 476, 528, 530, 485, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 458, 551, 479, 522, 503, 486, 510, 9, 5, 2, 0, 0, 0, 0, 0, 499, 512, 534, 517, 550, 519, 496, 506, 2, 0, 0, 0, 0, 0, 0, 0, 502, 532, 580, 500, 482, 456, 549, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4114, 4107, 4118, 4001, 4024, 4135, 4131] avg=4096.0000 max=4138 max_rank=0 min=4001 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 526, 497, 537, 499, 482, 492, 478, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 496, 539, 515, 523, 490, 536, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 548, 512, 476, 507, 507, 484, 29, 8, 0, 0, 0, 0, 0, 0, 532, 525, 540, 501, 535, 490, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 532, 496, 480, 551, 515, 520, 555, 527, 0, 0, 0, 0, 0, 0, 0, 0, 511, 479, 514, 495, 504, 535, 481, 512, 8, 5, 2, 0, 0, 0, 0, 0, 503, 527, 483, 518, 499, 504, 532, 505, 2, 0, 0, 0, 0, 0, 0, 0, 474, 546, 498, 544, 528, 473, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4133, 4111, 4125, 4176, 4046, 4073, 4085] avg=4096.0000 max=4176 max_rank=4 min=4019 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[538, 508, 487, 539, 517, 516, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 521, 521, 489, 488, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 500, 492, 528, 457, 537, 473, 535, 508, 30, 8, 0, 0, 0, 0, 0, 0, 513, 529, 524, 512, 486, 512, 515, 498, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 518, 495, 510, 494, 528, 534, 0, 0, 0, 0, 0, 0, 0, 0, 485, 530, 542, 511, 530, 516, 518, 500, 9, 6, 2, 0, 0, 0, 0, 0, 526, 493, 492, 526, 502, 518, 516, 534, 2, 0, 0, 0, 0, 0, 0, 0, 512, 510, 489, 486, 523, 494, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4094, 4068, 4089, 4087, 4149, 4109, 4028] avg=4096.0000 max=4149 max_rank=5 min=4028 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[573, 489, 549, 474, 522, 526, 541, 532, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 506, 482, 493, 505, 531, 482, 0, 0, 0, 0, 0, 0, 0, 0, 475, 508, 514, 481, 532, 511, 515, 541, 30, 7, 0, 0, 0, 0, 0, 0, 564, 524, 510, 504, 536, 480, 456, 522, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 517, 525, 515, 492, 549, 504, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 492, 511, 505, 496, 516, 482, 8, 5, 2, 0, 0, 0, 0, 0, 486, 498, 511, 538, 507, 498, 518, 540, 2, 0, 0, 0, 0, 0, 0, 0, 521, 475, 513, 536, 524, 493, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 3991, 4114, 4096, 4142, 3998, 4098, 4123] avg=4096.0000 max=4206 max_rank=0 min=3991 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 518, 515, 502, 514, 509, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 521, 514, 506, 539, 508, 518, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 545, 492, 519, 509, 515, 496, 470, 503, 34, 13, 0, 0, 0, 0, 0, 0, 467, 515, 479, 482, 562, 498, 519, 540, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 502, 505, 516, 512, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 531, 532, 531, 517, 566, 500, 485, 548, 12, 11, 5, 7, 0, 0, 0, 0, 545, 495, 496, 528, 494, 476, 512, 504, 8, 0, 0, 0, 0, 0, 0, 0, 525, 479, 522, 520, 498, 500, 518, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4131, 4096, 4062, 4039, 4245, 4058, 4105] avg=4096.0000 max=4245 max_rank=5 min=4032 min_rank=0 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 503, 469, 534, 501, 487, 529, 489, 0, 0, 0, 0, 0, 0, 0, 0, 546, 503, 507, 498, 514, 512, 503, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 497, 499, 501, 511, 504, 506, 32, 8, 0, 0, 0, 0, 0, 0, 475, 517, 525, 514, 538, 525, 542, 510, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 515, 522, 516, 512, 470, 510, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 502, 543, 525, 533, 519, 489, 8, 5, 2, 0, 0, 0, 0, 0, 498, 529, 525, 486, 516, 542, 507, 443, 2, 0, 0, 0, 0, 0, 0, 0, 532, 531, 529, 501, 538, 529, 498, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4091, 4076, 4146, 4081, 4139, 4048, 4161] avg=4096.0000 max=4161 max_rank=7 min=4026 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 524, 533, 523, 514, 513, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 522, 493, 472, 489, 533, 549, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 501, 517, 515, 520, 517, 505, 29, 8, 0, 0, 0, 0, 0, 0, 487, 492, 503, 541, 532, 502, 527, 459, 0, 0, 0, 0, 0, 0, 0, 0, 548, 511, 474, 545, 502, 519, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 547, 513, 481, 485, 523, 527, 526, 495, 9, 5, 2, 0, 0, 0, 0, 0, 511, 531, 507, 506, 513, 528, 506, 538, 3, 0, 0, 0, 0, 0, 0, 0, 459, 529, 488, 562, 504, 482, 466, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4109, 4127, 4043, 4133, 4113, 4143, 4000] avg=4096.0000 max=4143 max_rank=6 min=4000 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 527, 515, 521, 514, 509, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 529, 511, 511, 532, 529, 510, 487, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 515, 515, 483, 483, 516, 506, 30, 8, 0, 0, 0, 0, 0, 0, 485, 509, 513, 582, 440, 507, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 545, 435, 522, 508, 518, 539, 515, 0, 0, 0, 0, 0, 0, 0, 0, 531, 459, 512, 510, 525, 482, 491, 528, 9, 5, 2, 0, 0, 0, 0, 0, 521, 485, 521, 498, 507, 509, 548, 513, 2, 0, 0, 0, 0, 0, 0, 0, 469, 541, 549, 522, 518, 502, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4105, 4069, 4089, 4109, 4054, 4104, 4135] avg=4096.0000 max=4135 max_rank=7 min=4054 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[471, 487, 529, 526, 495, 530, 513, 519, 527, 512, 515, 530, 510, 509, 477, 480, 525, 535, 519, 502, 509, 524, 478, 494, 523, 510, 487, 511, 518, 498, 530, 542, 480, 531, 505, 515, 512, 519, 487, 513, 515, 505, 511, 502, 544, 489, 498, 557, 515, 489, 521, 531, 524, 522, 512, 535, 527, 502, 507, 511, 528, 505, 505, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[523, 507, 560, 477, 526, 493, 522, 521, 540, 523, 538, 494, 497, 544, 532, 500, 511, 493, 503, 492, 522, 526, 482, 486, 510, 497, 512, 500, 517, 492, 480, 509, 544, 482, 488, 517, 537, 527, 552, 512, 509, 508, 473, 499, 519, 511, 541, 546, 512, 523, 515, 493, 524, 494, 545, 540, 480, 515, 494, 516, 506, 496, 498, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[493, 482, 517, 461, 466, 501, 513, 503, 522, 490, 509, 505, 509, 536, 528, 513, 533, 504, 552, 492, 522, 496, 517, 521, 492, 504, 516, 502, 529, 511, 488, 515, 501, 513, 513, 521, 482, 530, 493, 504, 515, 509, 551, 508, 515, 540, 499, 544, 550, 501, 550, 513, 513, 495, 511, 516, 513, 526, 504, 558, 522, 529, 495, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[507, 534, 512, 509, 483, 496, 505, 482, 543, 522, 491, 504, 488, 539, 487, 528, 485, 507, 491, 537, 492, 528, 507, 546, 541, 484, 516, 543, 486, 499, 523, 490, 521, 525, 525, 519, 517, 509, 518, 532, 495, 549, 524, 470, 489, 482, 538, 494, 528, 554, 517, 516, 500, 528, 527, 503, 520, 484, 508, 499, 482, 517, 531, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[487, 483, 488, 513, 508, 531, 563, 480, 528, 518, 507, 534, 500, 485, 501, 512, 519, 496, 500, 522, 505, 531, 546, 502, 502, 489, 517, 508, 561, 495, 533, 512, 478, 532, 498, 488, 509, 494, 526, 547, 554, 484, 541, 512, 523, 502, 495, 555, 540, 539, 500, 535, 505, 520, 528, 513, 502, 520, 492, 516, 498, 516, 445, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[509, 520, 499, 543, 503, 525, 500, 499, 536, 496, 549, 517, 508, 503, 488, 487, 486, 519, 536, 516, 476, 515, 530, 481, 489, 536, 487, 497, 528, 506, 537, 478, 506, 536, 500, 535, 523, 493, 499, 532, 513, 544, 511, 529, 505, 523, 478, 525, 522, 486, 539, 493, 489, 521, 541, 491, 522, 513, 500, 518, 515, 507, 533, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[508, 530, 529, 533, 489, 482, 517, 508, 530, 507, 498, 520, 522, 507, 556, 500, 497, 508, 514, 535, 520, 523, 506, 489, 543, 505, 521, 495, 512, 513, 504, 518, 513, 483, 518, 478, 501, 514, 508, 510, 513, 486, 500, 518, 490, 495, 522, 550, 506, 523, 541, 487, 493, 499, 530, 540, 526, 506, 506, 541, 493, 505, 524, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[523, 512, 525, 536, 508, 503, 519, 502, 480, 506, 475, 478, 489, 542, 524, 516, 513, 502, 484, 500, 537, 524, 531, 523, 493, 535, 530, 536, 508, 521, 482, 496, 542, 498, 492, 486, 491, 507, 505, 490, 468, 497, 538, 516, 491, 553, 516, 511, 523, 514, 473, 496, 528, 512, 524, 526, 522, 541, 514, 528, 529, 540, 533, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[471, 487, 529, 526, 495, 530, 513, 519, 27, 0, 0, 0, 0, 0, 0, 0, 527, 512, 515, 530, 510, 509, 477, 480, 0, 0, 0, 0, 0, 0, 0, 0, 525, 535, 519, 502, 509, 524, 478, 494, 4, 1, 0, 0, 0, 0, 0, 0, 523, 510, 487, 511, 518, 498, 530, 542, 11, 1, 0, 0, 0, 0, 0, 0, 480, 531, 505, 515, 512, 519, 487, 513, 11, 0, 0, 0, 0, 0, 0, 0, 515, 505, 511, 502, 544, 489, 498, 541, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 521, 531, 524, 522, 474, 535, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 507, 510, 528, 505, 505, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4060, 4091, 4131, 4073, 4105, 4111, 4100] avg=4096.0000 max=4131 max_rank=3 min=4060 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 507, 560, 477, 526, 493, 522, 521, 29, 0, 0, 0, 0, 0, 0, 0, 540, 523, 538, 494, 497, 544, 532, 500, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 503, 492, 522, 526, 482, 486, 4, 1, 0, 0, 0, 0, 0, 0, 510, 497, 512, 500, 517, 492, 480, 509, 12, 1, 0, 0, 0, 0, 0, 0, 544, 482, 488, 517, 537, 527, 552, 512, 11, 0, 0, 0, 0, 0, 0, 0, 509, 508, 473, 499, 519, 511, 541, 530, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 515, 493, 524, 494, 504, 540, 0, 0, 0, 0, 0, 0, 0, 0, 480, 515, 494, 515, 506, 496, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4168, 4020, 4030, 4170, 4090, 4105, 4027] avg=4096.0000 max=4170 max_rank=4 min=4020 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 534, 512, 509, 483, 496, 505, 482, 28, 0, 0, 0, 0, 0, 0, 0, 543, 522, 491, 504, 488, 539, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 485, 507, 491, 537, 492, 528, 507, 546, 4, 1, 0, 0, 0, 0, 0, 0, 541, 484, 516, 543, 486, 499, 523, 490, 11, 1, 0, 0, 0, 0, 0, 0, 521, 525, 525, 519, 517, 509, 518, 532, 10, 0, 0, 0, 0, 0, 0, 0, 495, 549, 524, 470, 489, 482, 538, 479, 0, 0, 0, 0, 0, 0, 0, 0, 528, 554, 517, 516, 500, 528, 488, 503, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 508, 498, 482, 517, 531, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4102, 4098, 4094, 4176, 4026, 4134, 4082] avg=4096.0000 max=4176 max_rank=4 min=4026 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 520, 499, 543, 503, 525, 500, 499, 29, 0, 0, 0, 0, 0, 0, 0, 536, 496, 549, 517, 508, 503, 488, 487, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 536, 516, 476, 515, 530, 481, 4, 1, 0, 0, 0, 0, 0, 0, 489, 536, 487, 497, 528, 506, 537, 478, 12, 1, 0, 0, 0, 0, 0, 0, 506, 536, 500, 535, 523, 493, 499, 532, 11, 0, 0, 0, 0, 0, 0, 0, 513, 544, 511, 529, 505, 523, 478, 509, 0, 0, 0, 0, 0, 0, 0, 0, 522, 486, 539, 493, 489, 521, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 500, 517, 515, 507, 533, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4084, 4064, 4071, 4135, 4112, 4041, 4134] avg=4096.0000 max=4135 max_rank=4 min=4041 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 482, 517, 461, 466, 501, 513, 503, 27, 0, 0, 0, 0, 0, 0, 0, 522, 490, 509, 505, 509, 536, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 533, 504, 552, 492, 522, 496, 517, 521, 4, 1, 0, 0, 0, 0, 0, 0, 492, 504, 516, 502, 529, 511, 488, 515, 11, 1, 0, 0, 0, 0, 0, 0, 501, 513, 513, 521, 482, 530, 493, 504, 11, 0, 0, 0, 0, 0, 0, 0, 515, 509, 551, 508, 515, 540, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 550, 501, 550, 513, 513, 495, 473, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 504, 557, 522, 529, 495, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3963, 4112, 4142, 4069, 4068, 4165, 4111, 4138] avg=4096.0000 max=4165 max_rank=5 min=3963 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[487, 483, 488, 513, 508, 531, 563, 480, 28, 0, 0, 0, 0, 0, 0, 0, 528, 518, 507, 534, 500, 485, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 500, 522, 505, 531, 546, 502, 4, 1, 0, 0, 0, 0, 0, 0, 502, 489, 517, 508, 561, 495, 533, 512, 12, 1, 0, 0, 0, 0, 0, 0, 478, 532, 498, 488, 509, 494, 526, 547, 11, 0, 0, 0, 0, 0, 0, 0, 554, 484, 541, 512, 523, 502, 495, 539, 0, 0, 0, 0, 0, 0, 0, 0, 540, 539, 500, 535, 505, 520, 488, 513, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 492, 515, 498, 516, 445, 485, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4085, 4126, 4130, 4083, 4150, 4140, 3973] avg=4096.0000 max=4150 max_rank=5 min=3973 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[508, 530, 529, 533, 489, 482, 517, 508, 34, 0, 0, 0, 0, 0, 0, 0, 530, 507, 498, 520, 522, 507, 556, 500, 7, 0, 0, 0, 0, 0, 0, 0, 497, 508, 514, 535, 520, 523, 506, 489, 9, 7, 0, 0, 0, 0, 0, 0, 543, 505, 521, 495, 512, 513, 504, 518, 16, 3, 0, 0, 0, 0, 0, 0, 513, 483, 518, 478, 501, 514, 508, 510, 17, 0, 0, 0, 0, 0, 0, 0, 513, 486, 500, 518, 490, 495, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 541, 487, 493, 499, 480, 540, 0, 0, 0, 0, 0, 0, 0, 0, 526, 506, 506, 527, 493, 505, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4147, 4108, 4130, 4042, 4045, 4069, 4097] avg=4096.0000 max=4147 max_rank=1 min=4042 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 512, 525, 536, 508, 503, 519, 502, 28, 0, 0, 0, 0, 0, 0, 0, 480, 506, 475, 478, 489, 542, 524, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 502, 484, 500, 537, 524, 531, 523, 4, 1, 0, 0, 0, 0, 0, 0, 493, 535, 530, 536, 508, 521, 482, 496, 11, 1, 0, 0, 0, 0, 0, 0, 542, 498, 492, 486, 491, 507, 505, 490, 10, 0, 0, 0, 0, 0, 0, 0, 468, 497, 538, 516, 491, 553, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 523, 514, 473, 496, 528, 512, 485, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 541, 514, 527, 529, 540, 533, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4010, 4119, 4113, 4021, 4075, 4057, 4217] avg=4096.0000 max=4217 max_rank=7 min=4010 min_rank=1 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[514, 501, 486, 533, 522, 509, 490, 468, 508, 516, 456, 469, 531, 528, 546, 504, 474, 468, 540, 510, 532, 533, 500, 546, 526, 512, 505, 544, 532, 510, 512, 551, 505, 535, 509, 509, 535, 508, 515, 508, 536, 500, 510, 450, 516, 534, 521, 499, 509, 486, 509, 471, 480, 473, 546, 509, 531, 528, 519, 513, 570, 518, 528, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[527, 531, 509, 502, 518, 535, 486, 504, 496, 540, 498, 481, 518, 514, 494, 520, 521, 558, 486, 500, 497, 511, 506, 546, 523, 481, 495, 520, 517, 505, 515, 506, 490, 531, 522, 537, 495, 471, 510, 515, 552, 537, 479, 500, 526, 526, 535, 539, 479, 536, 498, 530, 511, 500, 509, 527, 483, 522, 527, 527, 502, 492, 507, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[546, 500, 511, 504, 496, 497, 528, 509, 506, 537, 518, 513, 539, 524, 498, 501, 543, 492, 544, 537, 531, 500, 502, 508, 503, 535, 503, 517, 499, 500, 519, 547, 539, 506, 507, 499, 519, 535, 520, 502, 536, 497, 516, 528, 511, 466, 554, 504, 469, 487, 502, 499, 503, 515, 456, 490, 522, 540, 514, 501, 495, 514, 511, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[475, 512, 503, 482, 526, 526, 526, 518, 545, 517, 497, 508, 515, 488, 519, 552, 520, 513, 498, 502, 513, 547, 528, 476, 496, 519, 538, 530, 502, 504, 533, 506, 464, 542, 516, 558, 544, 488, 509, 499, 497, 533, 555, 517, 514, 526, 521, 513, 492, 525, 459, 526, 520, 481, 496, 484, 539, 501, 501, 518, 495, 489, 522, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[491, 508, 534, 537, 520, 526, 526, 520, 485, 500, 514, 537, 475, 521, 515, 528, 522, 499, 504, 488, 519, 505, 550, 491, 489, 523, 544, 523, 499, 537, 531, 511, 523, 510, 505, 522, 528, 524, 505, 504, 539, 496, 489, 470, 539, 505, 518, 531, 523, 516, 506, 523, 521, 498, 486, 495, 514, 515, 517, 513, 472, 469, 496, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[513, 477, 516, 533, 456, 526, 531, 473, 513, 505, 517, 501, 507, 534, 530, 532, 462, 533, 525, 515, 486, 499, 511, 532, 482, 535, 497, 526, 507, 506, 518, 501, 492, 481, 510, 510, 511, 534, 503, 480, 507, 524, 526, 506, 466, 517, 490, 531, 525, 509, 510, 522, 549, 548, 524, 500, 533, 544, 532, 488, 510, 518, 538, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[471, 510, 526, 512, 517, 510, 492, 534, 537, 507, 501, 505, 546, 559, 509, 505, 504, 483, 505, 517, 493, 496, 500, 508, 495, 520, 497, 500, 543, 495, 511, 511, 517, 488, 513, 536, 483, 516, 496, 478, 469, 528, 505, 531, 569, 527, 552, 499, 512, 528, 504, 527, 529, 491, 548, 526, 526, 512, 522, 523, 518, 480, 475, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[520, 496, 537, 500, 517, 529, 488, 543, 492, 498, 496, 522, 474, 515, 538, 543, 534, 478, 476, 522, 501, 517, 502, 546, 525, 475, 556, 526, 531, 534, 514, 496, 517, 522, 499, 481, 502, 504, 508, 486, 531, 477, 543, 520, 519, 552, 526, 520, 497, 527, 503, 501, 520, 486, 547, 509, 460, 531, 509, 502, 478, 487, 510, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 531, 509, 502, 518, 535, 486, 504, 9, 0, 0, 0, 0, 0, 0, 0, 496, 540, 498, 481, 518, 514, 494, 511, 0, 0, 0, 0, 0, 0, 0, 0, 521, 558, 486, 500, 497, 511, 506, 546, 7, 0, 0, 0, 0, 0, 0, 0, 523, 481, 495, 497, 517, 505, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 490, 531, 522, 537, 495, 471, 510, 515, 14, 2, 0, 0, 0, 0, 0, 0, 552, 537, 479, 500, 526, 526, 498, 539, 0, 0, 0, 0, 0, 0, 0, 0, 479, 536, 498, 530, 511, 500, 509, 527, 35, 0, 0, 0, 0, 0, 0, 0, 483, 522, 527, 527, 502, 492, 507, 493, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4052, 4132, 4039, 4087, 4157, 4125, 4055] avg=4096.0000 max=4157 max_rank=5 min=4039 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[546, 500, 511, 504, 496, 497, 528, 509, 9, 0, 0, 0, 0, 0, 0, 0, 506, 537, 518, 513, 539, 524, 498, 492, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 544, 537, 531, 500, 502, 508, 7, 0, 0, 0, 0, 0, 0, 0, 503, 535, 503, 494, 499, 500, 519, 547, 0, 0, 0, 0, 0, 0, 0, 0, 539, 506, 507, 499, 519, 535, 520, 502, 14, 2, 0, 0, 0, 0, 0, 0, 536, 497, 516, 528, 511, 466, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 469, 487, 502, 499, 503, 515, 456, 490, 36, 0, 0, 0, 0, 0, 0, 0, 522, 540, 514, 501, 495, 514, 511, 504, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4127, 4164, 4100, 4143, 4074, 3957, 4103] avg=4096.0000 max=4164 max_rank=2 min=3957 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 501, 486, 533, 522, 509, 490, 468, 10, 0, 0, 0, 0, 0, 0, 0, 508, 516, 456, 469, 531, 528, 546, 495, 0, 0, 0, 0, 0, 0, 0, 0, 474, 468, 540, 510, 532, 533, 500, 546, 7, 0, 0, 0, 0, 0, 0, 0, 526, 512, 505, 519, 532, 510, 512, 551, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 509, 509, 535, 508, 515, 508, 15, 2, 0, 0, 0, 0, 0, 0, 536, 500, 510, 450, 516, 534, 485, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 509, 471, 480, 473, 546, 509, 34, 0, 0, 0, 0, 0, 0, 0, 531, 528, 519, 513, 570, 518, 528, 512, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4049, 4110, 4167, 4141, 4030, 4017, 4221] avg=4096.0000 max=4221 max_rank=7 min=4017 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 508, 534, 537, 520, 526, 526, 520, 9, 0, 0, 0, 0, 0, 0, 0, 485, 500, 514, 537, 475, 521, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 522, 499, 504, 488, 519, 505, 550, 491, 7, 0, 0, 0, 0, 0, 0, 0, 489, 523, 544, 500, 499, 537, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 505, 522, 528, 524, 505, 504, 14, 2, 0, 0, 0, 0, 0, 0, 539, 496, 489, 470, 539, 505, 482, 531, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 506, 523, 521, 498, 486, 495, 34, 0, 0, 0, 0, 0, 0, 0, 514, 515, 517, 513, 472, 469, 496, 524, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4066, 4085, 4134, 4137, 4051, 4102, 4022] avg=4096.0000 max=4171 max_rank=0 min=4022 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[513, 477, 516, 533, 456, 526, 531, 473, 9, 0, 0, 0, 0, 0, 0, 0, 513, 505, 517, 501, 507, 534, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 462, 533, 525, 515, 486, 499, 511, 532, 8, 0, 0, 0, 0, 0, 0, 0, 482, 535, 497, 503, 507, 506, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0, 492, 481, 510, 510, 511, 534, 503, 480, 14, 2, 0, 0, 0, 0, 0, 0, 507, 524, 526, 506, 466, 517, 456, 531, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 510, 522, 549, 548, 524, 500, 32, 0, 0, 0, 0, 0, 0, 0, 533, 544, 532, 488, 510, 518, 538, 531, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4129, 4071, 4049, 4037, 4033, 4219, 4196] avg=4096.0000 max=4219 max_rank=6 min=4033 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[475, 512, 503, 482, 526, 526, 526, 518, 9, 0, 0, 0, 0, 0, 0, 0, 545, 517, 497, 508, 515, 488, 519, 542, 0, 0, 0, 0, 0, 0, 0, 0, 520, 513, 498, 502, 513, 547, 528, 476, 8, 0, 0, 0, 0, 0, 0, 0, 496, 519, 538, 507, 502, 504, 533, 506, 0, 0, 0, 0, 0, 0, 0, 0, 464, 542, 516, 558, 544, 488, 509, 499, 14, 2, 0, 0, 0, 0, 0, 0, 497, 533, 555, 517, 514, 526, 485, 513, 0, 0, 0, 0, 0, 0, 0, 0, 492, 525, 459, 526, 520, 481, 496, 484, 34, 0, 0, 0, 0, 0, 0, 0, 539, 501, 501, 518, 495, 489, 522, 490, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4131, 4105, 4105, 4136, 4140, 4017, 4057] avg=4096.0000 max=4140 max_rank=5 min=4017 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[520, 496, 537, 500, 517, 529, 488, 543, 14, 7, 0, 0, 0, 0, 0, 0, 492, 498, 496, 522, 474, 515, 538, 519, 0, 0, 0, 0, 0, 0, 0, 0, 534, 478, 476, 522, 501, 517, 502, 546, 12, 0, 0, 0, 0, 0, 0, 0, 525, 475, 556, 493, 531, 534, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 499, 481, 502, 504, 508, 486, 19, 7, 0, 0, 0, 0, 0, 0, 531, 477, 543, 520, 519, 552, 479, 520, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 503, 501, 520, 486, 547, 509, 40, 0, 0, 0, 0, 0, 0, 0, 460, 531, 509, 502, 478, 487, 510, 553, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4054, 4088, 4124, 4045, 4141, 4130, 4035] avg=4096.0000 max=4151 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[471, 510, 526, 512, 517, 510, 492, 534, 9, 0, 0, 0, 0, 0, 0, 0, 537, 507, 501, 505, 546, 559, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 504, 483, 505, 517, 493, 496, 500, 508, 7, 0, 0, 0, 0, 0, 0, 0, 495, 520, 497, 478, 543, 495, 511, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 488, 513, 536, 483, 516, 496, 478, 13, 2, 0, 0, 0, 0, 0, 0, 469, 528, 505, 531, 569, 527, 514, 499, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 504, 527, 529, 491, 548, 526, 36, 0, 0, 0, 0, 0, 0, 0, 526, 512, 522, 523, 518, 480, 475, 521, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4160, 4013, 4050, 4042, 4142, 4201, 4079] avg=4096.0000 max=4201 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[498, 536, 491, 514, 495, 493, 490, 531, 506, 489, 502, 525, 519, 518, 547, 507, 542, 522, 502, 541, 501, 508, 503, 528, 495, 453, 500, 509, 516, 528, 523, 502, 504, 521, 533, 548, 499, 518, 502, 516, 528, 527, 529, 515, 541, 528, 506, 517, 492, 514, 514, 468, 533, 558, 491, 511, 507, 508, 508, 481, 523, 483, 498, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[544, 517, 501, 510, 508, 490, 544, 519, 500, 556, 529, 498, 509, 513, 504, 487, 526, 504, 499, 514, 486, 492, 510, 539, 498, 497, 522, 505, 529, 520, 525, 542, 479, 488, 481, 509, 499, 523, 576, 500, 499, 503, 519, 470, 515, 501, 547, 496, 506, 532, 562, 509, 560, 487, 506, 512, 513, 513, 505, 491, 500, 533, 514, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[540, 474, 482, 518, 492, 510, 537, 521, 545, 504, 509, 505, 532, 518, 511, 488, 531, 510, 493, 520, 519, 523, 493, 503, 524, 519, 511, 541, 511, 530, 470, 561, 501, 533, 502, 502, 469, 540, 526, 533, 492, 511, 519, 501, 504, 508, 532, 514, 509, 513, 500, 481, 528, 487, 507, 518, 499, 484, 531, 535, 486, 514, 532, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[484, 485, 500, 522, 499, 505, 491, 512, 530, 459, 513, 486, 469, 500, 535, 501, 516, 510, 539, 511, 542, 483, 504, 533, 535, 547, 510, 534, 486, 541, 522, 522, 473, 546, 511, 542, 543, 493, 514, 515, 492, 522, 519, 485, 514, 524, 495, 480, 556, 495, 508, 499, 508, 521, 513, 531, 531, 490, 509, 512, 519, 546, 523, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[507, 521, 503, 512, 508, 533, 511, 509, 546, 507, 534, 524, 522, 488, 515, 512, 544, 504, 518, 508, 518, 541, 510, 502, 508, 508, 499, 513, 487, 493, 527, 505, 480, 483, 531, 517, 517, 501, 555, 482, 496, 507, 545, 531, 511, 499, 518, 507, 528, 489, 519, 442, 530, 506, 536, 515, 509, 509, 527, 497, 503, 536, 493, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[484, 522, 496, 488, 516, 525, 503, 552, 500, 515, 499, 521, 483, 505, 511, 522, 496, 501, 542, 526, 539, 528, 503, 498, 502, 494, 476, 501, 523, 516, 516, 527, 485, 483, 510, 509, 538, 472, 511, 521, 561, 513, 513, 523, 459, 532, 505, 503, 529, 541, 542, 534, 529, 506, 534, 505, 519, 539, 461, 494, 496, 504, 521, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[483, 522, 513, 498, 538, 502, 544, 459, 531, 496, 540, 498, 522, 516, 520, 507, 504, 488, 549, 510, 520, 531, 500, 455, 496, 524, 518, 499, 492, 514, 522, 518, 496, 529, 517, 541, 497, 488, 476, 521, 539, 541, 530, 489, 505, 520, 497, 520, 472, 504, 509, 532, 538, 543, 491, 542, 536, 511, 496, 500, 525, 513, 536, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[513, 513, 528, 498, 525, 536, 531, 512, 525, 544, 502, 545, 483, 535, 515, 493, 500, 512, 495, 531, 522, 479, 504, 504, 539, 510, 507, 483, 515, 483, 502, 508, 498, 511, 526, 518, 527, 545, 469, 524, 507, 534, 501, 511, 496, 512, 523, 489, 542, 512, 519, 542, 502, 488, 508, 543, 486, 524, 517, 508, 516, 496, 511, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[498, 536, 491, 514, 495, 493, 490, 531, 8, 5, 0, 0, 0, 0, 0, 0, 504, 489, 502, 525, 519, 518, 547, 507, 0, 0, 0, 0, 0, 0, 0, 0, 542, 522, 502, 534, 501, 508, 503, 528, 0, 0, 0, 0, 0, 0, 0, 0, 495, 453, 500, 509, 516, 528, 523, 502, 1, 0, 0, 0, 0, 0, 0, 0, 504, 521, 533, 548, 499, 518, 502, 516, 2, 2, 0, 0, 0, 0, 0, 0, 528, 527, 528, 515, 541, 528, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 514, 468, 505, 558, 491, 511, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 508, 481, 523, 483, 498, 513, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4111, 4140, 4027, 4145, 4190, 4053, 4041] avg=4096.0000 max=4190 max_rank=5 min=4027 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[540, 474, 482, 518, 492, 510, 537, 521, 8, 4, 0, 0, 0, 0, 0, 0, 543, 504, 509, 505, 532, 518, 511, 488, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 493, 514, 519, 523, 493, 503, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 511, 541, 511, 530, 470, 561, 1, 0, 0, 0, 0, 0, 0, 0, 501, 533, 502, 502, 469, 540, 526, 533, 2, 2, 0, 0, 0, 0, 0, 0, 492, 511, 518, 501, 504, 508, 532, 514, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 500, 481, 500, 487, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 499, 484, 531, 535, 486, 514, 532, 512, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4110, 4086, 4168, 4110, 4080, 4015, 4113] avg=4096.0000 max=4168 max_rank=3 min=4015 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[544, 517, 501, 510, 508, 490, 544, 519, 9, 4, 0, 0, 0, 0, 0, 0, 498, 556, 529, 498, 509, 513, 504, 487, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 499, 508, 486, 492, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 522, 505, 529, 520, 525, 542, 1, 0, 0, 0, 0, 0, 0, 0, 479, 488, 481, 509, 499, 523, 576, 500, 2, 2, 0, 0, 0, 0, 0, 0, 499, 503, 518, 470, 515, 501, 547, 496, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 562, 509, 530, 487, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 505, 491, 500, 533, 514, 483, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4094, 4064, 4139, 4059, 4049, 4144, 4073] avg=4096.0000 max=4146 max_rank=0 min=4049 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 485, 500, 522, 499, 505, 491, 512, 8, 4, 0, 0, 0, 0, 0, 0, 528, 459, 513, 486, 469, 500, 535, 501, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 539, 505, 542, 483, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 535, 547, 510, 534, 486, 541, 522, 522, 1, 0, 0, 0, 0, 0, 0, 0, 473, 546, 511, 542, 543, 493, 514, 515, 2, 2, 0, 0, 0, 0, 0, 0, 492, 522, 518, 485, 514, 524, 495, 480, 0, 0, 0, 0, 0, 0, 0, 0, 556, 495, 508, 499, 481, 521, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 490, 509, 512, 519, 546, 523, 513, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 3991, 4132, 4198, 4141, 4030, 4104, 4162] avg=4096.0000 max=4198 max_rank=3 min=3991 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 521, 503, 512, 508, 533, 511, 509, 8, 4, 0, 0, 0, 0, 0, 0, 544, 507, 534, 524, 522, 488, 515, 512, 0, 0, 0, 0, 0, 0, 0, 0, 544, 504, 518, 502, 518, 541, 510, 502, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 499, 513, 487, 493, 527, 505, 1, 0, 0, 0, 0, 0, 0, 0, 480, 483, 531, 517, 517, 501, 555, 482, 2, 2, 0, 0, 0, 0, 0, 0, 496, 507, 544, 531, 511, 499, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 519, 442, 502, 506, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 527, 497, 503, 536, 493, 512, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4146, 4139, 4041, 4070, 4113, 4037, 4106] avg=4096.0000 max=4146 max_rank=1 min=4037 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[483, 522, 513, 498, 538, 502, 544, 459, 9, 4, 0, 0, 0, 0, 0, 0, 529, 496, 540, 498, 522, 516, 520, 507, 0, 0, 0, 0, 0, 0, 0, 0, 504, 488, 549, 504, 520, 531, 500, 455, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 518, 499, 492, 514, 522, 518, 1, 0, 0, 0, 0, 0, 0, 0, 496, 529, 517, 541, 497, 488, 476, 521, 2, 2, 0, 0, 0, 0, 0, 0, 539, 541, 529, 489, 505, 520, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 472, 504, 509, 532, 509, 543, 491, 542, 0, 0, 0, 0, 0, 0, 0, 0, 536, 511, 496, 500, 525, 513, 536, 485, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4128, 4051, 4084, 4069, 4140, 4102, 4122] avg=4096.0000 max=4140 max_rank=5 min=4051 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 522, 496, 488, 516, 525, 503, 552, 8, 4, 0, 0, 0, 0, 0, 0, 498, 515, 499, 521, 483, 505, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 542, 520, 539, 528, 503, 498, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 476, 501, 523, 516, 516, 527, 1, 0, 0, 0, 0, 0, 0, 0, 485, 483, 510, 509, 538, 472, 511, 521, 2, 2, 0, 0, 0, 0, 0, 0, 561, 513, 512, 523, 459, 532, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 542, 534, 501, 506, 534, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 539, 461, 494, 496, 504, 521, 546, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4054, 4127, 4056, 4033, 4108, 4192, 4100] avg=4096.0000 max=4192 max_rank=6 min=4033 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 513, 528, 498, 525, 536, 531, 512, 13, 10, 0, 0, 0, 0, 0, 0, 517, 544, 502, 545, 483, 535, 515, 493, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 495, 513, 522, 479, 504, 504, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 507, 483, 515, 483, 502, 508, 8, 0, 0, 0, 0, 0, 0, 0, 498, 511, 526, 518, 527, 545, 469, 524, 8, 8, 7, 0, 0, 0, 0, 0, 507, 534, 486, 511, 496, 512, 523, 489, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 519, 542, 467, 488, 508, 543, 0, 0, 0, 0, 0, 0, 0, 0, 486, 524, 517, 508, 516, 496, 511, 471, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4134, 4029, 4055, 4141, 4058, 4121, 4051] avg=4096.0000 max=4179 max_rank=0 min=4029 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[525, 526, 481, 587, 512, 510, 468, 486, 443, 490, 516, 532, 501, 495, 500, 516, 539, 515, 549, 501, 504, 513, 509, 494, 527, 476, 534, 508, 521, 514, 511, 507, 481, 560, 518, 512, 527, 508, 511, 521, 512, 521, 517, 516, 493, 514, 498, 497, 520, 529, 520, 553, 512, 515, 465, 543, 496, 478, 545, 490, 500, 513, 550, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[527, 521, 524, 508, 553, 493, 492, 559, 483, 522, 520, 555, 528, 501, 512, 508, 522, 498, 472, 484, 484, 487, 506, 508, 493, 544, 516, 515, 531, 493, 528, 509, 501, 505, 496, 504, 491, 503, 496, 530, 504, 494, 549, 515, 533, 531, 525, 535, 515, 510, 518, 498, 486, 485, 522, 535, 534, 504, 534, 509, 509, 502, 521, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[532, 548, 514, 468, 514, 497, 512, 500, 509, 488, 532, 520, 501, 497, 497, 519, 499, 513, 490, 494, 520, 561, 502, 492, 525, 513, 509, 507, 488, 522, 530, 512, 468, 523, 540, 556, 518, 557, 473, 468, 542, 509, 545, 515, 514, 485, 525, 517, 494, 515, 529, 566, 495, 530, 514, 487, 494, 510, 480, 507, 530, 505, 538, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[498, 520, 532, 506, 531, 493, 495, 515, 505, 509, 517, 532, 500, 523, 496, 511, 471, 512, 515, 521, 512, 526, 516, 511, 506, 497, 554, 526, 568, 544, 510, 496, 476, 478, 527, 518, 511, 505, 556, 469, 536, 545, 518, 560, 485, 475, 474, 494, 492, 544, 520, 548, 500, 512, 484, 502, 479, 506, 526, 548, 480, 482, 529, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[486, 531, 523, 518, 520, 467, 528, 533, 497, 472, 559, 528, 506, 481, 526, 491, 521, 518, 532, 528, 494, 517, 527, 505, 505, 495, 525, 525, 489, 494, 488, 494, 504, 525, 525, 505, 521, 505, 516, 508, 563, 508, 473, 501, 531, 534, 538, 533, 502, 491, 513, 525, 511, 503, 478, 535, 473, 540, 480, 525, 506, 515, 505, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[510, 526, 526, 499, 524, 517, 539, 525, 501, 527, 507, 503, 493, 503, 540, 484, 542, 524, 479, 510, 498, 506, 515, 507, 507, 522, 500, 524, 513, 498, 468, 528, 523, 499, 521, 530, 477, 534, 476, 533, 484, 510, 523, 456, 500, 526, 533, 524, 534, 495, 534, 484, 517, 493, 533, 531, 520, 512, 536, 500, 512, 528, 505, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[484, 524, 510, 531, 508, 511, 522, 504, 494, 510, 551, 535, 475, 498, 527, 528, 533, 490, 526, 495, 525, 472, 526, 501, 492, 542, 505, 485, 468, 508, 505, 537, 484, 531, 518, 547, 547, 497, 501, 552, 521, 532, 526, 539, 513, 509, 506, 498, 517, 492, 495, 487, 513, 528, 493, 515, 522, 489, 512, 508, 523, 534, 509, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[526, 507, 493, 523, 509, 519, 527, 490, 501, 518, 496, 490, 506, 527, 505, 530, 518, 509, 521, 488, 500, 513, 514, 517, 485, 512, 525, 520, 527, 527, 540, 493, 512, 497, 507, 517, 555, 473, 500, 531, 468, 527, 502, 482, 509, 519, 531, 529, 475, 565, 542, 493, 508, 515, 522, 520, 526, 493, 498, 490, 526, 514, 507, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 509, 481, 587, 512, 510, 468, 486, 0, 0, 0, 0, 0, 0, 0, 0, 443, 490, 516, 532, 501, 495, 500, 516, 21, 1, 0, 0, 0, 0, 0, 0, 539, 515, 549, 501, 504, 513, 509, 494, 16, 3, 0, 0, 0, 0, 0, 0, 527, 476, 533, 508, 521, 514, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 518, 511, 527, 508, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 496, 516, 493, 514, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 529, 515, 553, 512, 515, 465, 543, 0, 0, 0, 0, 0, 0, 0, 0, 496, 478, 545, 490, 500, 513, 550, 523, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4015, 4143, 4097, 4137, 4047, 4152, 4099] avg=4096.0000 max=4152 max_rank=6 min=4015 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[532, 530, 514, 468, 514, 497, 512, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 488, 532, 520, 501, 497, 497, 519, 22, 1, 0, 0, 0, 0, 0, 0, 499, 513, 490, 494, 520, 561, 502, 492, 17, 3, 0, 0, 0, 0, 0, 0, 525, 513, 508, 507, 488, 522, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 468, 523, 540, 555, 518, 557, 473, 468, 0, 0, 0, 0, 0, 0, 0, 0, 542, 509, 523, 515, 514, 485, 525, 517, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 524, 566, 495, 530, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 480, 507, 530, 505, 538, 494, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4086, 4091, 4105, 4102, 4130, 4125, 4062] avg=4096.0000 max=4130 max_rank=5 min=4062 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 505, 524, 508, 553, 493, 492, 559, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 520, 555, 528, 501, 512, 508, 22, 0, 0, 0, 0, 0, 0, 0, 522, 498, 472, 484, 484, 487, 506, 508, 16, 3, 0, 0, 0, 0, 0, 0, 493, 544, 515, 515, 531, 493, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 496, 503, 491, 503, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 527, 515, 533, 531, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 513, 498, 486, 485, 522, 535, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 534, 509, 509, 502, 521, 478, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4151, 3980, 4128, 4025, 4164, 4064, 4095] avg=4096.0000 max=4164 max_rank=5 min=3980 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 509, 526, 499, 524, 517, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 501, 527, 507, 503, 493, 503, 540, 484, 21, 1, 0, 0, 0, 0, 0, 0, 542, 524, 479, 510, 498, 506, 515, 507, 16, 3, 0, 0, 0, 0, 0, 0, 507, 522, 499, 524, 513, 498, 468, 528, 0, 0, 0, 0, 0, 0, 0, 0, 523, 499, 521, 529, 477, 534, 476, 533, 0, 0, 0, 0, 0, 0, 0, 0, 484, 510, 502, 456, 500, 526, 533, 524, 0, 0, 0, 0, 0, 0, 0, 0, 534, 495, 529, 484, 517, 493, 533, 531, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 536, 500, 512, 528, 505, 520, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4080, 4100, 4059, 4092, 4035, 4116, 4137] avg=4096.0000 max=4149 max_rank=0 min=4035 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 504, 532, 506, 531, 493, 495, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 517, 532, 500, 523, 496, 511, 21, 0, 0, 0, 0, 0, 0, 0, 471, 512, 515, 521, 512, 526, 516, 511, 16, 3, 0, 0, 0, 0, 0, 0, 506, 497, 553, 526, 568, 544, 510, 496, 0, 0, 0, 0, 0, 0, 0, 0, 476, 478, 527, 517, 511, 505, 556, 469, 0, 0, 0, 0, 0, 0, 0, 0, 536, 545, 497, 560, 485, 475, 474, 494, 0, 0, 0, 0, 0, 0, 0, 0, 492, 544, 515, 548, 500, 512, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 506, 526, 548, 480, 482, 529, 521, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4114, 4103, 4200, 4039, 4066, 4097, 4075] avg=4096.0000 max=4200 max_rank=3 min=4039 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 514, 523, 518, 520, 467, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0, 497, 472, 559, 528, 506, 481, 526, 491, 19, 1, 0, 0, 0, 0, 0, 0, 521, 518, 532, 528, 494, 517, 527, 505, 16, 3, 0, 0, 0, 0, 0, 0, 505, 495, 524, 525, 489, 494, 488, 494, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 525, 504, 521, 505, 516, 508, 0, 0, 0, 0, 0, 0, 0, 0, 563, 508, 454, 501, 531, 534, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0, 502, 491, 508, 525, 511, 503, 478, 535, 0, 0, 0, 0, 0, 0, 0, 0, 473, 540, 480, 525, 506, 515, 505, 553, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4080, 4161, 4014, 4108, 4162, 4053, 4101] avg=4096.0000 max=4162 max_rank=5 min=4014 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 501, 510, 531, 508, 511, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 551, 535, 475, 498, 527, 528, 27, 4, 0, 0, 0, 0, 0, 0, 533, 490, 526, 495, 525, 472, 526, 501, 19, 5, 0, 0, 0, 0, 0, 0, 492, 542, 500, 485, 468, 508, 505, 537, 0, 0, 0, 0, 0, 0, 0, 0, 484, 531, 518, 544, 547, 497, 501, 552, 0, 0, 0, 0, 0, 0, 0, 0, 521, 532, 499, 539, 513, 509, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 481, 487, 513, 528, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 512, 508, 523, 534, 509, 488, 9, 5, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4149, 4092, 4037, 4174, 4117, 4026, 4102] avg=4096.0000 max=4174 max_rank=4 min=4026 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[526, 492, 493, 523, 509, 519, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 496, 490, 506, 527, 505, 530, 20, 0, 0, 0, 0, 0, 0, 0, 518, 509, 521, 488, 500, 513, 514, 517, 15, 3, 0, 0, 0, 0, 0, 0, 485, 512, 524, 520, 527, 527, 540, 493, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 507, 516, 555, 473, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 468, 527, 482, 482, 509, 519, 531, 529, 0, 0, 0, 0, 0, 0, 0, 0, 475, 565, 537, 493, 508, 515, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 526, 493, 498, 490, 526, 514, 507, 539, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4093, 4098, 4128, 4091, 4047, 4135, 4097] avg=4096.0000 max=4135 max_rank=6 min=4047 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 15:43:13] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 272.7 | throughput per GPU (TFLOP/s/GPU): 452.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.231299E+00 | loss scale: 1.0 | grad norm: 5.048 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[496, 512, 519, 489, 535, 524, 495, 509, 534, 492, 511, 531, 533, 513, 509, 497, 520, 556, 505, 524, 526, 537, 487, 507, 514, 494, 532, 493, 502, 508, 510, 500, 518, 487, 492, 528, 494, 546, 506, 493, 522, 520, 515, 512, 519, 537, 499, 526, 518, 482, 520, 495, 484, 533, 506, 507, 539, 480, 528, 530, 524, 498, 487, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[493, 520, 522, 527, 499, 528, 508, 509, 509, 541, 528, 537, 488, 522, 524, 519, 529, 538, 498, 494, 487, 548, 485, 503, 532, 515, 486, 537, 543, 549, 518, 498, 495, 528, 495, 495, 496, 503, 446, 546, 521, 504, 503, 518, 490, 471, 528, 496, 518, 522, 531, 518, 535, 515, 483, 546, 520, 493, 527, 497, 501, 514, 486, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[537, 513, 456, 468, 529, 510, 458, 493, 487, 520, 551, 515, 478, 519, 518, 500, 519, 502, 500, 511, 507, 499, 484, 507, 482, 522, 522, 527, 525, 543, 489, 456, 518, 553, 508, 515, 525, 520, 532, 515, 544, 460, 544, 534, 544, 491, 491, 520, 509, 513, 517, 526, 513, 532, 561, 505, 514, 530, 540, 477, 511, 527, 515, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[537, 496, 469, 469, 492, 481, 508, 546, 465, 514, 493, 539, 489, 486, 529, 482, 516, 502, 524, 532, 482, 498, 513, 501, 526, 519, 530, 546, 531, 519, 535, 528, 488, 521, 495, 488, 509, 538, 523, 550, 509, 491, 489, 522, 524, 504, 489, 483, 515, 531, 487, 565, 523, 515, 540, 520, 495, 562, 531, 531, 497, 471, 540, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[519, 523, 506, 517, 513, 521, 501, 536, 504, 521, 508, 487, 569, 487, 472, 491, 515, 482, 509, 518, 489, 526, 506, 518, 546, 476, 538, 477, 494, 485, 556, 530, 526, 505, 497, 522, 504, 521, 513, 511, 550, 474, 526, 467, 485, 528, 508, 527, 560, 511, 505, 492, 510, 501, 528, 532, 517, 503, 515, 532, 528, 513, 518, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[518, 485, 524, 544, 549, 521, 492, 500, 530, 496, 516, 519, 476, 480, 500, 510, 483, 508, 515, 543, 505, 503, 490, 495, 482, 533, 508, 507, 532, 531, 526, 507, 498, 500, 502, 538, 515, 533, 522, 553, 494, 513, 539, 549, 499, 509, 512, 522, 512, 497, 526, 467, 503, 529, 484, 513, 532, 468, 498, 511, 517, 508, 519, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[509, 523, 502, 492, 489, 540, 507, 526, 479, 501, 507, 493, 492, 530, 497, 542, 493, 507, 535, 521, 550, 499, 489, 449, 530, 527, 533, 534, 499, 493, 542, 514, 505, 508, 492, 540, 514, 524, 522, 543, 503, 527, 525, 517, 526, 487, 504, 487, 518, 502, 527, 505, 499, 501, 485, 557, 527, 507, 537, 501, 517, 485, 516, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[544, 505, 507, 525, 518, 523, 516, 478, 495, 478, 520, 502, 532, 518, 528, 498, 514, 559, 523, 513, 541, 544, 471, 508, 540, 466, 481, 493, 484, 491, 505, 513, 495, 532, 493, 529, 478, 525, 534, 528, 524, 522, 517, 533, 524, 529, 478, 480, 536, 515, 540, 494, 481, 505, 499, 509, 533, 518, 480, 544, 552, 528, 476, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[496, 512, 519, 489, 535, 524, 495, 509, 9, 8, 0, 0, 0, 0, 0, 0, 534, 492, 511, 531, 533, 513, 509, 497, 19, 6, 0, 0, 0, 0, 0, 0, 520, 556, 505, 524, 526, 537, 487, 507, 7, 5, 0, 0, 0, 0, 0, 0, 514, 494, 532, 493, 502, 508, 495, 500, 0, 0, 0, 0, 0, 0, 0, 0, 518, 487, 492, 528, 494, 546, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 515, 512, 519, 537, 499, 526, 8, 0, 0, 0, 0, 0, 0, 0, 518, 482, 520, 495, 484, 533, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 524, 480, 528, 530, 524, 498, 487, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4145, 4174, 4038, 4051, 4158, 4026, 4080] avg=4096.0000 max=4174 max_rank=2 min=4026 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[537, 496, 469, 469, 492, 481, 508, 546, 10, 8, 0, 0, 0, 0, 0, 0, 465, 514, 493, 539, 489, 486, 529, 482, 19, 7, 0, 0, 0, 0, 0, 0, 516, 502, 524, 532, 482, 498, 513, 501, 6, 6, 0, 0, 0, 0, 0, 0, 526, 519, 530, 546, 531, 519, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 488, 521, 495, 488, 509, 538, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 489, 522, 524, 504, 489, 483, 7, 0, 0, 0, 0, 0, 0, 0, 515, 531, 487, 565, 523, 515, 540, 501, 0, 0, 0, 0, 0, 0, 0, 0, 482, 562, 531, 531, 497, 471, 540, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4023, 4080, 4217, 4098, 4018, 4177, 4139] avg=4096.0000 max=4217 max_rank=3 min=4016 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[519, 523, 506, 517, 513, 521, 501, 536, 10, 8, 0, 0, 0, 0, 0, 0, 504, 521, 508, 487, 569, 487, 472, 491, 20, 7, 0, 0, 0, 0, 0, 0, 515, 482, 509, 518, 489, 526, 506, 518, 6, 6, 0, 0, 0, 0, 0, 0, 546, 476, 538, 477, 494, 485, 539, 530, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 497, 522, 504, 521, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 550, 474, 526, 467, 485, 528, 508, 527, 7, 0, 0, 0, 0, 0, 0, 0, 560, 511, 505, 492, 510, 501, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 515, 532, 528, 513, 518, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4066, 4075, 4085, 4085, 4072, 4119, 4112] avg=4096.0000 max=4154 max_rank=0 min=4066 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 522, 527, 499, 528, 508, 509, 9, 8, 0, 0, 0, 0, 0, 0, 509, 541, 528, 537, 488, 522, 524, 519, 20, 7, 0, 0, 0, 0, 0, 0, 529, 538, 498, 494, 487, 548, 485, 503, 6, 6, 0, 0, 0, 0, 0, 0, 532, 515, 486, 537, 543, 549, 502, 498, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 495, 495, 496, 503, 446, 532, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 503, 518, 490, 471, 528, 496, 7, 0, 0, 0, 0, 0, 0, 0, 518, 522, 531, 518, 535, 515, 483, 526, 0, 0, 0, 0, 0, 0, 0, 0, 507, 493, 527, 497, 501, 514, 486, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4195, 4094, 4162, 3990, 4038, 4148, 4018] avg=4096.0000 max=4195 max_rank=1 min=3990 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 523, 502, 492, 489, 540, 507, 526, 10, 8, 0, 0, 0, 0, 0, 0, 479, 501, 507, 493, 492, 530, 497, 542, 21, 7, 0, 0, 0, 0, 0, 0, 493, 507, 535, 521, 550, 499, 489, 449, 6, 6, 0, 0, 0, 0, 0, 0, 530, 527, 533, 534, 499, 493, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 492, 540, 514, 524, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 525, 517, 526, 487, 504, 487, 7, 0, 0, 0, 0, 0, 0, 0, 518, 502, 527, 505, 499, 501, 485, 536, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 537, 501, 517, 485, 516, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4069, 4055, 4155, 4134, 4083, 4073, 4093] avg=4096.0000 max=4155 max_rank=3 min=4055 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[537, 513, 456, 468, 529, 510, 458, 493, 9, 8, 0, 0, 0, 0, 0, 0, 487, 520, 551, 515, 478, 519, 518, 500, 19, 6, 0, 0, 0, 0, 0, 0, 519, 502, 500, 511, 507, 499, 484, 507, 6, 6, 0, 0, 0, 0, 0, 0, 482, 522, 522, 527, 525, 543, 474, 456, 0, 0, 0, 0, 0, 0, 0, 0, 518, 553, 508, 515, 525, 520, 532, 501, 0, 0, 0, 0, 0, 0, 0, 0, 544, 460, 544, 534, 544, 491, 491, 520, 7, 0, 0, 0, 0, 0, 0, 0, 509, 513, 517, 526, 513, 532, 561, 486, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 540, 477, 511, 527, 515, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3981, 4113, 4041, 4051, 4172, 4135, 4157, 4118] avg=4096.0000 max=4172 max_rank=4 min=3981 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[544, 505, 507, 525, 518, 523, 516, 478, 13, 12, 0, 0, 0, 0, 0, 0, 495, 478, 520, 502, 532, 518, 528, 498, 23, 10, 0, 0, 0, 0, 0, 0, 514, 559, 523, 513, 541, 544, 471, 508, 11, 10, 0, 0, 0, 0, 0, 0, 540, 466, 481, 493, 484, 491, 482, 513, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 493, 529, 478, 525, 534, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 517, 533, 524, 529, 478, 480, 12, 0, 0, 0, 0, 0, 0, 0, 536, 515, 540, 494, 481, 505, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 480, 544, 552, 528, 476, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4104, 4194, 3950, 4092, 4119, 4056, 4112] avg=4096.0000 max=4194 max_rank=2 min=3950 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 485, 524, 544, 549, 521, 492, 500, 9, 9, 0, 0, 0, 0, 0, 0, 530, 496, 516, 519, 476, 480, 500, 510, 19, 7, 0, 0, 0, 0, 0, 0, 483, 508, 515, 543, 505, 503, 490, 495, 7, 6, 0, 0, 0, 0, 0, 0, 482, 533, 508, 507, 532, 531, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 502, 538, 515, 533, 522, 538, 0, 0, 0, 0, 0, 0, 0, 0, 494, 513, 539, 549, 499, 509, 512, 522, 8, 0, 0, 0, 0, 0, 0, 0, 512, 497, 526, 467, 503, 529, 484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 517, 468, 498, 511, 517, 508, 519, 558, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4053, 4055, 4110, 4146, 4145, 4012, 4096] avg=4096.0000 max=4151 max_rank=0 min=4012 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[529, 484, 498, 512, 540, 493, 534, 500, 483, 496, 552, 494, 518, 538, 495, 501, 514, 523, 484, 496, 514, 540, 511, 506, 507, 513, 549, 517, 502, 486, 503, 513, 507, 507, 517, 552, 536, 502, 515, 514, 518, 502, 512, 516, 505, 489, 542, 508, 510, 485, 521, 515, 562, 513, 532, 537, 475, 467, 506, 499, 509, 520, 512, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[514, 507, 506, 501, 564, 520, 554, 526, 536, 495, 483, 536, 535, 507, 508, 548, 485, 537, 462, 511, 504, 532, 490, 534, 534, 527, 456, 509, 522, 464, 512, 527, 506, 488, 506, 517, 527, 506, 510, 494, 511, 514, 528, 505, 520, 504, 546, 533, 468, 501, 527, 516, 506, 492, 512, 505, 492, 515, 510, 503, 542, 524, 515, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[496, 503, 542, 498, 525, 480, 537, 502, 482, 466, 545, 513, 520, 505, 489, 511, 520, 553, 524, 495, 541, 506, 474, 537, 494, 462, 505, 511, 501, 539, 517, 522, 519, 549, 500, 530, 486, 502, 489, 498, 490, 510, 520, 509, 507, 517, 514, 542, 531, 510, 522, 510, 530, 492, 501, 551, 532, 538, 519, 488, 507, 542, 510, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[522, 513, 526, 496, 476, 515, 476, 484, 527, 517, 516, 490, 502, 533, 513, 525, 499, 514, 524, 537, 564, 488, 536, 514, 516, 544, 509, 550, 502, 520, 504, 517, 506, 492, 501, 497, 534, 528, 489, 519, 518, 473, 536, 516, 503, 475, 489, 503, 523, 553, 506, 545, 512, 511, 503, 515, 505, 526, 510, 490, 511, 514, 492, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[474, 521, 523, 521, 534, 509, 516, 521, 493, 530, 484, 500, 520, 507, 496, 515, 514, 523, 519, 546, 492, 488, 487, 536, 515, 507, 501, 478, 519, 516, 491, 529, 518, 514, 512, 500, 496, 533, 560, 486, 483, 514, 514, 486, 496, 550, 512, 508, 503, 509, 502, 525, 528, 497, 499, 508, 523, 509, 536, 498, 529, 511, 528, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[514, 516, 489, 551, 514, 485, 498, 528, 513, 509, 513, 483, 545, 527, 500, 501, 485, 528, 535, 498, 558, 476, 539, 481, 514, 504, 485, 519, 524, 523, 524, 506, 509, 542, 468, 491, 526, 542, 551, 488, 529, 473, 502, 514, 514, 545, 501, 497, 531, 469, 507, 513, 526, 554, 554, 515, 477, 517, 512, 517, 507, 498, 510, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[518, 512, 470, 537, 526, 527, 504, 487, 490, 515, 480, 496, 519, 491, 517, 516, 503, 533, 523, 515, 507, 515, 512, 488, 505, 532, 494, 506, 529, 502, 469, 522, 479, 540, 535, 509, 505, 502, 577, 577, 520, 529, 504, 545, 535, 474, 531, 498, 539, 474, 515, 470, 497, 513, 525, 524, 539, 532, 535, 488, 524, 479, 523, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[538, 480, 513, 508, 486, 509, 525, 494, 511, 525, 529, 516, 493, 507, 511, 532, 490, 523, 491, 508, 521, 530, 531, 529, 540, 534, 519, 501, 492, 551, 475, 495, 510, 529, 506, 515, 517, 517, 442, 552, 523, 486, 486, 503, 491, 480, 503, 557, 518, 515, 518, 513, 526, 510, 516, 503, 494, 549, 501, 505, 530, 506, 488, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[474, 521, 523, 521, 534, 509, 516, 521, 5, 0, 0, 0, 0, 0, 0, 0, 493, 530, 484, 500, 520, 507, 496, 515, 13, 0, 0, 0, 0, 0, 0, 0, 514, 508, 519, 546, 492, 488, 487, 536, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 501, 478, 519, 516, 491, 529, 7, 4, 0, 0, 0, 0, 0, 0, 518, 500, 512, 500, 496, 533, 560, 486, 0, 0, 0, 0, 0, 0, 0, 0, 483, 514, 514, 486, 496, 550, 512, 508, 11, 0, 0, 0, 0, 0, 0, 0, 503, 509, 502, 525, 508, 497, 499, 508, 0, 0, 0, 0, 0, 0, 0, 0, 523, 509, 536, 498, 529, 511, 528, 556, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4058, 4090, 4067, 4105, 4074, 4051, 4199] avg=4096.0000 max=4199 max_rank=7 min=4051 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 507, 506, 501, 564, 520, 554, 526, 5, 0, 0, 0, 0, 0, 0, 0, 536, 495, 483, 536, 535, 507, 508, 548, 12, 0, 0, 0, 0, 0, 0, 0, 485, 522, 462, 511, 504, 532, 490, 534, 0, 0, 0, 0, 0, 0, 0, 0, 534, 527, 456, 509, 522, 464, 512, 527, 7, 4, 0, 0, 0, 0, 0, 0, 506, 475, 506, 517, 527, 506, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 528, 505, 520, 504, 546, 533, 11, 0, 0, 0, 0, 0, 0, 0, 468, 501, 527, 516, 487, 492, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 510, 503, 542, 524, 515, 479, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 4160, 4040, 4062, 4041, 4172, 4008, 4088] avg=4096.0000 max=4197 max_rank=0 min=4008 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 503, 542, 498, 525, 480, 537, 502, 6, 0, 0, 0, 0, 0, 0, 0, 482, 466, 545, 513, 520, 505, 489, 511, 13, 0, 0, 0, 0, 0, 0, 0, 520, 538, 524, 495, 541, 506, 474, 537, 0, 0, 0, 0, 0, 0, 0, 0, 494, 462, 505, 511, 501, 539, 517, 522, 7, 4, 0, 0, 0, 0, 0, 0, 519, 533, 500, 530, 486, 502, 489, 498, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 520, 509, 507, 517, 514, 542, 11, 0, 0, 0, 0, 0, 0, 0, 531, 510, 522, 510, 510, 492, 501, 551, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 519, 488, 507, 542, 510, 488, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4044, 4135, 4062, 4057, 4120, 4127, 4134] avg=4096.0000 max=4135 max_rank=2 min=4044 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 489, 551, 514, 485, 498, 528, 6, 0, 0, 0, 0, 0, 0, 0, 513, 509, 513, 483, 545, 527, 500, 501, 13, 0, 0, 0, 0, 0, 0, 0, 485, 513, 535, 498, 558, 476, 539, 481, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 485, 519, 524, 523, 524, 506, 7, 4, 0, 0, 0, 0, 0, 0, 509, 527, 468, 491, 526, 542, 551, 488, 0, 0, 0, 0, 0, 0, 0, 0, 529, 473, 502, 514, 514, 545, 501, 497, 11, 0, 0, 0, 0, 0, 0, 0, 531, 469, 507, 513, 506, 554, 554, 515, 0, 0, 0, 0, 0, 0, 0, 0, 477, 517, 512, 517, 507, 498, 510, 484, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4104, 4085, 4110, 4102, 4086, 4149, 4031] avg=4096.0000 max=4149 max_rank=6 min=4031 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[529, 484, 498, 512, 540, 493, 534, 500, 5, 0, 0, 0, 0, 0, 0, 0, 483, 496, 552, 494, 518, 538, 495, 501, 14, 0, 0, 0, 0, 0, 0, 0, 514, 508, 484, 496, 514, 540, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 549, 517, 502, 486, 503, 513, 8, 4, 0, 0, 0, 0, 0, 0, 507, 493, 517, 552, 536, 502, 515, 514, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 512, 516, 505, 489, 542, 508, 11, 0, 0, 0, 0, 0, 0, 0, 510, 485, 521, 515, 540, 513, 532, 537, 0, 0, 0, 0, 0, 0, 0, 0, 475, 467, 506, 499, 509, 520, 512, 518, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4091, 4073, 4102, 4136, 4103, 4153, 4015] avg=4096.0000 max=4153 max_rank=6 min=4015 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[522, 513, 526, 496, 476, 515, 476, 484, 9, 0, 0, 0, 0, 0, 0, 0, 527, 517, 516, 490, 502, 533, 513, 525, 15, 0, 0, 0, 0, 0, 0, 0, 499, 494, 524, 537, 564, 488, 536, 514, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 509, 550, 502, 520, 504, 517, 10, 5, 0, 0, 0, 0, 0, 0, 506, 470, 501, 497, 534, 528, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 518, 473, 536, 516, 503, 475, 489, 503, 13, 0, 0, 0, 0, 0, 0, 0, 523, 553, 506, 545, 487, 511, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 526, 510, 490, 511, 514, 492, 504, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4138, 4156, 4177, 4044, 4026, 4143, 4067] avg=4096.0000 max=4177 max_rank=3 min=4017 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[538, 480, 513, 508, 486, 509, 525, 494, 5, 0, 0, 0, 0, 0, 0, 0, 511, 525, 529, 516, 493, 507, 511, 532, 13, 0, 0, 0, 0, 0, 0, 0, 490, 508, 491, 508, 521, 530, 531, 529, 0, 0, 0, 0, 0, 0, 0, 0, 540, 534, 519, 501, 492, 551, 475, 495, 7, 4, 0, 0, 0, 0, 0, 0, 510, 515, 506, 515, 517, 517, 442, 552, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 486, 503, 491, 480, 503, 557, 11, 0, 0, 0, 0, 0, 0, 0, 518, 515, 518, 513, 506, 510, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 494, 549, 501, 505, 530, 506, 488, 552, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4137, 4108, 4118, 4074, 4040, 4099, 4134] avg=4096.0000 max=4137 max_rank=1 min=4040 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 512, 470, 537, 526, 527, 504, 487, 6, 0, 0, 0, 0, 0, 0, 0, 490, 515, 480, 496, 519, 491, 517, 516, 12, 0, 0, 0, 0, 0, 0, 0, 503, 518, 523, 515, 507, 515, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0, 505, 532, 494, 506, 529, 502, 469, 522, 7, 4, 0, 0, 0, 0, 0, 0, 479, 525, 535, 509, 505, 502, 577, 577, 0, 0, 0, 0, 0, 0, 0, 0, 520, 529, 504, 545, 535, 474, 531, 498, 11, 0, 0, 0, 0, 0, 0, 0, 539, 474, 515, 470, 478, 513, 525, 524, 0, 0, 0, 0, 0, 0, 0, 0, 539, 532, 535, 488, 524, 479, 523, 471, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4036, 4081, 4070, 4209, 4147, 4038, 4100] avg=4096.0000 max=4209 max_rank=4 min=4036 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[518, 526, 515, 523, 517, 497, 523, 513, 511, 529, 507, 530, 496, 504, 519, 504, 487, 509, 501, 501, 541, 515, 508, 515, 501, 514, 541, 538, 517, 516, 539, 512, 501, 521, 473, 514, 472, 516, 506, 538, 529, 508, 496, 494, 489, 539, 452, 526, 509, 502, 478, 511, 543, 513, 532, 495, 525, 501, 505, 498, 521, 553, 520, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[517, 534, 498, 499, 522, 504, 541, 505, 505, 513, 504, 491, 512, 524, 492, 519, 490, 526, 503, 502, 539, 496, 527, 538, 515, 497, 500, 542, 500, 472, 547, 516, 502, 551, 499, 509, 519, 508, 525, 522, 481, 511, 479, 492, 494, 473, 512, 490, 555, 512, 518, 556, 462, 510, 534, 513, 522, 530, 538, 525, 509, 514, 506, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[494, 516, 508, 505, 497, 548, 582, 502, 512, 493, 489, 530, 497, 545, 493, 487, 525, 553, 515, 475, 535, 503, 556, 513, 507, 513, 500, 480, 533, 535, 505, 504, 483, 523, 469, 507, 461, 516, 498, 515, 517, 492, 526, 512, 505, 525, 494, 527, 517, 512, 516, 489, 544, 527, 479, 560, 507, 503, 505, 532, 543, 478, 510, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[557, 521, 530, 511, 525, 495, 540, 467, 506, 525, 537, 501, 529, 535, 498, 521, 475, 531, 525, 508, 509, 560, 520, 494, 460, 536, 531, 529, 505, 494, 501, 532, 526, 527, 500, 526, 564, 509, 470, 479, 529, 505, 489, 473, 497, 493, 481, 543, 495, 521, 527, 513, 501, 489, 498, 533, 532, 534, 471, 528, 473, 492, 552, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[561, 516, 497, 495, 536, 505, 504, 517, 506, 488, 540, 509, 522, 526, 523, 493, 503, 513, 509, 521, 518, 473, 555, 487, 500, 512, 509, 504, 530, 473, 509, 507, 493, 505, 517, 460, 544, 525, 503, 506, 537, 513, 531, 527, 495, 513, 497, 515, 542, 486, 491, 536, 517, 478, 525, 509, 486, 506, 525, 526, 473, 551, 525, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[490, 524, 510, 543, 532, 486, 512, 496, 515, 514, 515, 535, 539, 531, 492, 534, 487, 557, 502, 524, 519, 438, 524, 490, 531, 528, 536, 541, 479, 501, 480, 522, 543, 502, 590, 517, 482, 496, 504, 490, 493, 497, 493, 500, 529, 496, 528, 517, 484, 482, 498, 517, 544, 523, 502, 519, 499, 526, 514, 494, 501, 520, 531, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[487, 521, 520, 528, 515, 531, 486, 514, 502, 489, 521, 475, 543, 501, 500, 528, 526, 524, 495, 515, 521, 507, 530, 469, 489, 477, 485, 515, 550, 551, 514, 542, 481, 492, 514, 498, 522, 520, 534, 499, 519, 531, 475, 506, 490, 527, 490, 532, 490, 509, 512, 523, 507, 516, 566, 517, 459, 538, 531, 511, 510, 541, 528, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[538, 477, 517, 484, 528, 504, 522, 534, 501, 507, 523, 522, 512, 534, 459, 496, 500, 497, 533, 524, 485, 525, 486, 529, 537, 461, 502, 575, 538, 535, 527, 504, 556, 497, 542, 499, 494, 495, 526, 502, 494, 476, 533, 500, 528, 486, 511, 502, 521, 506, 495, 502, 496, 514, 506, 493, 538, 516, 535, 483, 526, 551, 545, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 526, 515, 523, 517, 497, 498, 513, 0, 0, 0, 0, 0, 0, 0, 0, 511, 529, 507, 530, 496, 504, 519, 504, 1, 0, 0, 0, 0, 0, 0, 0, 487, 508, 501, 501, 541, 515, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 541, 522, 517, 516, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 473, 514, 472, 516, 506, 538, 16, 2, 2, 0, 0, 0, 0, 0, 529, 508, 496, 494, 489, 539, 452, 526, 27, 23, 0, 0, 0, 0, 0, 0, 509, 502, 478, 509, 543, 513, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0, 525, 501, 505, 498, 521, 553, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4101, 4076, 4162, 4061, 4083, 4081, 4097] avg=4096.0000 max=4162 max_rank=3 min=4061 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[557, 521, 530, 511, 525, 495, 514, 467, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 537, 501, 529, 535, 498, 521, 1, 0, 0, 0, 0, 0, 0, 0, 475, 530, 525, 508, 509, 560, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 460, 536, 531, 513, 505, 494, 501, 532, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 500, 526, 564, 509, 470, 479, 16, 2, 2, 0, 0, 0, 0, 0, 529, 505, 489, 473, 497, 493, 481, 543, 29, 24, 0, 0, 0, 0, 0, 0, 495, 521, 527, 511, 501, 489, 498, 533, 0, 0, 0, 0, 0, 0, 0, 0, 532, 534, 471, 528, 473, 492, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4153, 4121, 4072, 4121, 4063, 4075, 4043] avg=4096.0000 max=4153 max_rank=1 min=4043 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[538, 477, 517, 484, 528, 504, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 523, 522, 512, 534, 459, 496, 1, 0, 0, 0, 0, 0, 0, 0, 500, 496, 533, 524, 485, 525, 486, 529, 0, 0, 0, 0, 0, 0, 0, 0, 537, 461, 502, 558, 538, 535, 527, 504, 0, 0, 0, 0, 0, 0, 0, 0, 556, 497, 542, 499, 494, 495, 526, 502, 17, 2, 2, 0, 0, 0, 0, 0, 494, 476, 533, 500, 528, 486, 511, 502, 29, 23, 0, 0, 0, 0, 0, 0, 521, 506, 495, 500, 496, 514, 506, 493, 0, 0, 0, 0, 0, 0, 0, 0, 538, 516, 535, 483, 526, 551, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4055, 4078, 4162, 4132, 4082, 4031, 4149] avg=4096.0000 max=4162 max_rank=3 min=4031 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 534, 498, 499, 522, 504, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 504, 491, 512, 524, 492, 519, 1, 0, 0, 0, 0, 0, 0, 0, 490, 525, 503, 502, 539, 496, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 500, 526, 500, 472, 547, 516, 0, 0, 0, 0, 0, 0, 0, 0, 502, 551, 499, 509, 519, 508, 525, 522, 16, 2, 2, 0, 0, 0, 0, 0, 481, 511, 479, 492, 494, 473, 512, 490, 26, 24, 0, 0, 0, 0, 0, 0, 555, 512, 518, 554, 462, 510, 534, 513, 0, 0, 0, 0, 0, 0, 0, 0, 522, 530, 538, 525, 509, 514, 480, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4061, 4120, 4073, 4155, 3982, 4158, 4125] avg=4096.0000 max=4158 max_rank=6 min=3982 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 516, 508, 505, 497, 548, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 489, 530, 497, 545, 493, 487, 6, 2, 0, 0, 0, 0, 0, 0, 525, 547, 515, 475, 535, 503, 556, 513, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 500, 463, 533, 535, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 483, 523, 469, 507, 461, 516, 498, 515, 17, 9, 6, 0, 0, 0, 0, 0, 517, 492, 526, 512, 505, 525, 494, 527, 30, 30, 0, 0, 0, 0, 0, 0, 517, 512, 516, 481, 544, 527, 479, 560, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 505, 532, 543, 478, 480, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4054, 4169, 4060, 4004, 4158, 4136, 4074] avg=4096.0000 max=4169 max_rank=2 min=4004 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[561, 516, 497, 495, 536, 505, 480, 517, 0, 0, 0, 0, 0, 0, 0, 0, 506, 488, 540, 509, 522, 526, 523, 493, 1, 0, 0, 0, 0, 0, 0, 0, 503, 512, 509, 521, 518, 473, 555, 487, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 509, 489, 530, 473, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 517, 460, 544, 525, 503, 506, 15, 2, 2, 0, 0, 0, 0, 0, 537, 513, 531, 527, 495, 513, 497, 515, 28, 22, 0, 0, 0, 0, 0, 0, 542, 486, 491, 534, 517, 478, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 486, 506, 525, 526, 473, 551, 497, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4108, 4078, 4029, 4072, 4178, 4082, 4114] avg=4096.0000 max=4178 max_rank=5 min=4029 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[490, 524, 510, 543, 532, 486, 488, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 515, 535, 539, 531, 492, 534, 1, 0, 0, 0, 0, 0, 0, 0, 487, 556, 502, 524, 519, 438, 524, 490, 0, 0, 0, 0, 0, 0, 0, 0, 531, 528, 536, 525, 479, 501, 480, 522, 0, 0, 0, 0, 0, 0, 0, 0, 543, 502, 590, 517, 482, 496, 504, 490, 16, 2, 2, 0, 0, 0, 0, 0, 493, 497, 493, 500, 529, 496, 528, 517, 28, 22, 0, 0, 0, 0, 0, 0, 484, 482, 498, 515, 544, 523, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 499, 526, 514, 494, 501, 520, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4176, 4040, 4102, 4144, 4103, 4067, 4067] avg=4096.0000 max=4176 max_rank=1 min=4040 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[487, 521, 520, 528, 515, 531, 463, 514, 0, 0, 0, 0, 0, 0, 0, 0, 502, 489, 521, 475, 543, 501, 500, 528, 1, 0, 0, 0, 0, 0, 0, 0, 526, 523, 495, 515, 521, 507, 530, 469, 0, 0, 0, 0, 0, 0, 0, 0, 489, 477, 485, 500, 550, 551, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 481, 492, 514, 498, 522, 520, 534, 499, 15, 2, 2, 0, 0, 0, 0, 0, 519, 531, 475, 506, 490, 527, 490, 532, 28, 21, 0, 0, 0, 0, 0, 0, 490, 509, 512, 521, 507, 516, 566, 517, 0, 0, 0, 0, 0, 0, 0, 0, 459, 538, 531, 511, 510, 541, 500, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4060, 4086, 4108, 4079, 4119, 4138, 4099] avg=4096.0000 max=4138 max_rank=6 min=4060 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[495, 520, 507, 460, 481, 511, 531, 534, 570, 482, 496, 536, 546, 503, 534, 530, 493, 466, 519, 514, 508, 546, 536, 507, 527, 499, 502, 532, 502, 499, 500, 486, 489, 487, 530, 489, 539, 483, 522, 525, 487, 532, 508, 530, 539, 508, 504, 528, 511, 535, 514, 519, 496, 508, 515, 501, 531, 521, 515, 500, 502, 510, 497, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[502, 509, 539, 484, 504, 523, 490, 543, 512, 477, 537, 510, 485, 501, 487, 523, 529, 510, 511, 538, 509, 527, 515, 532, 489, 520, 516, 500, 507, 551, 504, 490, 509, 505, 509, 493, 461, 530, 483, 513, 512, 522, 538, 501, 488, 509, 561, 524, 536, 528, 514, 517, 539, 472, 540, 485, 506, 543, 507, 499, 529, 503, 511, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[545, 531, 538, 531, 497, 494, 498, 520, 527, 506, 489, 463, 497, 502, 519, 510, 543, 528, 511, 491, 496, 484, 525, 522, 522, 504, 519, 492, 513, 515, 475, 502, 513, 515, 512, 533, 545, 521, 510, 519, 481, 513, 503, 497, 521, 554, 553, 567, 511, 501, 506, 515, 503, 508, 502, 474, 533, 487, 499, 495, 505, 511, 546, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[532, 519, 509, 498, 489, 520, 510, 473, 482, 509, 512, 550, 501, 521, 491, 530, 502, 546, 509, 518, 517, 539, 527, 539, 531, 528, 541, 495, 556, 521, 487, 490, 514, 529, 514, 527, 505, 492, 494, 497, 528, 524, 501, 496, 495, 530, 522, 515, 491, 510, 515, 486, 523, 512, 509, 508, 512, 499, 482, 520, 518, 491, 505, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[524, 512, 499, 501, 479, 526, 499, 511, 486, 479, 492, 513, 539, 517, 508, 519, 519, 519, 555, 504, 519, 552, 557, 527, 515, 514, 512, 512, 543, 510, 515, 503, 504, 520, 524, 549, 497, 542, 506, 498, 518, 535, 494, 516, 485, 482, 507, 527, 470, 562, 512, 511, 483, 514, 510, 514, 492, 503, 526, 470, 466, 485, 544, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[524, 497, 519, 525, 505, 516, 543, 514, 539, 495, 503, 540, 517, 484, 542, 533, 508, 554, 518, 511, 537, 529, 527, 516, 542, 518, 530, 499, 547, 516, 473, 509, 517, 497, 511, 515, 491, 482, 523, 486, 534, 517, 512, 527, 515, 479, 463, 500, 482, 492, 548, 477, 506, 494, 483, 506, 530, 493, 522, 484, 534, 502, 523, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[511, 508, 482, 543, 523, 524, 515, 534, 502, 503, 526, 515, 496, 496, 505, 506, 516, 533, 491, 507, 526, 523, 525, 520, 460, 519, 511, 488, 532, 502, 505, 522, 564, 524, 502, 535, 504, 504, 464, 525, 517, 504, 533, 502, 499, 562, 529, 453, 541, 496, 519, 485, 526, 509, 487, 512, 507, 488, 499, 543, 515, 517, 506, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[523, 470, 511, 538, 556, 507, 541, 510, 463, 504, 499, 534, 523, 523, 494, 487, 526, 486, 501, 533, 486, 504, 474, 528, 536, 516, 498, 551, 521, 508, 515, 486, 505, 502, 505, 495, 494, 520, 462, 529, 505, 535, 535, 516, 533, 512, 533, 515, 517, 494, 553, 503, 506, 527, 524, 520, 493, 501, 506, 472, 528, 514, 535, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[489, 520, 507, 460, 481, 511, 531, 534, 0, 0, 0, 0, 0, 0, 0, 0, 570, 482, 496, 536, 546, 503, 534, 530, 14, 4, 0, 0, 0, 0, 0, 0, 493, 466, 519, 514, 508, 493, 536, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 502, 532, 502, 499, 500, 486, 2, 0, 0, 0, 0, 0, 0, 0, 489, 487, 530, 489, 539, 483, 522, 525, 13, 8, 0, 0, 0, 0, 0, 0, 487, 505, 508, 530, 539, 508, 504, 528, 0, 0, 0, 0, 0, 0, 0, 0, 511, 535, 514, 519, 496, 508, 515, 501, 23, 0, 0, 0, 0, 0, 0, 0, 531, 521, 515, 500, 502, 510, 497, 521, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4215, 4036, 4049, 4085, 4109, 4122, 4119] avg=4096.0000 max=4215 max_rank=1 min=4033 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[495, 509, 539, 484, 504, 523, 490, 543, 0, 0, 0, 0, 0, 0, 0, 0, 512, 477, 537, 510, 485, 501, 487, 523, 14, 4, 0, 0, 0, 0, 0, 0, 529, 510, 511, 538, 509, 476, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 520, 516, 500, 507, 551, 504, 490, 3, 0, 0, 0, 0, 0, 0, 0, 509, 505, 509, 493, 461, 530, 483, 513, 12, 7, 0, 0, 0, 0, 0, 0, 512, 496, 538, 501, 488, 509, 561, 524, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 514, 517, 539, 472, 540, 485, 22, 0, 0, 0, 0, 0, 0, 0, 506, 543, 507, 499, 529, 503, 511, 507, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4050, 4120, 4080, 4022, 4129, 4153, 4127] avg=4096.0000 max=4153 max_rank=6 min=4022 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[538, 531, 538, 531, 497, 494, 498, 520, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 489, 463, 497, 502, 519, 510, 13, 4, 0, 0, 0, 0, 0, 0, 543, 528, 511, 491, 496, 437, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 519, 492, 513, 515, 475, 502, 3, 0, 0, 0, 0, 0, 0, 0, 513, 515, 512, 533, 545, 521, 510, 519, 12, 7, 0, 0, 0, 0, 0, 0, 481, 488, 503, 497, 521, 554, 553, 567, 0, 0, 0, 0, 0, 0, 0, 0, 511, 501, 506, 515, 503, 508, 502, 474, 20, 0, 0, 0, 0, 0, 0, 0, 533, 487, 499, 495, 505, 511, 546, 506, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4030, 4053, 4045, 4187, 4164, 4040, 4102] avg=4096.0000 max=4187 max_rank=4 min=4030 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 519, 509, 498, 489, 520, 510, 473, 0, 0, 0, 0, 0, 0, 0, 0, 482, 509, 512, 550, 501, 521, 491, 530, 14, 4, 0, 0, 0, 0, 0, 0, 502, 546, 509, 518, 517, 487, 527, 539, 0, 0, 0, 0, 0, 0, 0, 0, 531, 528, 541, 495, 556, 521, 487, 490, 3, 0, 0, 0, 0, 0, 0, 0, 514, 529, 514, 527, 505, 492, 494, 497, 12, 7, 0, 0, 0, 0, 0, 0, 528, 498, 501, 496, 495, 530, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 491, 510, 515, 486, 523, 512, 509, 508, 23, 0, 0, 0, 0, 0, 0, 0, 512, 499, 482, 520, 518, 491, 505, 512, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4114, 4145, 4152, 4091, 4085, 4077, 4061] avg=4096.0000 max=4152 max_rank=3 min=4043 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 497, 519, 525, 505, 516, 543, 514, 0, 0, 0, 0, 0, 0, 0, 0, 539, 495, 503, 540, 517, 484, 542, 533, 14, 4, 0, 0, 0, 0, 0, 0, 508, 554, 518, 511, 537, 478, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 530, 499, 547, 516, 473, 509, 3, 0, 0, 0, 0, 0, 0, 0, 517, 497, 511, 515, 491, 482, 523, 486, 12, 7, 0, 0, 0, 0, 0, 0, 534, 491, 512, 527, 515, 479, 463, 500, 0, 0, 0, 0, 0, 0, 0, 0, 482, 492, 548, 477, 506, 494, 483, 506, 22, 0, 0, 0, 0, 0, 0, 0, 530, 493, 522, 484, 534, 502, 523, 493, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4171, 4149, 4137, 4041, 4021, 4010, 4103] avg=4096.0000 max=4171 max_rank=1 min=4010 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16175 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 512, 499, 501, 479, 526, 499, 511, 0, 0, 0, 0, 0, 0, 0, 0, 486, 479, 492, 513, 539, 517, 508, 519, 14, 4, 0, 0, 0, 0, 0, 0, 519, 519, 555, 504, 519, 498, 557, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 512, 512, 543, 510, 515, 503, 3, 0, 0, 0, 0, 0, 0, 0, 504, 520, 524, 549, 497, 542, 506, 498, 13, 8, 0, 0, 0, 0, 0, 0, 518, 508, 494, 516, 485, 482, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 470, 562, 512, 511, 483, 514, 510, 514, 23, 0, 0, 0, 0, 0, 0, 0, 492, 503, 526, 470, 466, 485, 544, 522, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4071, 4198, 4127, 4161, 4037, 4099, 4031] avg=4096.0000 max=4198 max_rank=2 min=4031 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 508, 482, 543, 523, 524, 515, 534, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 526, 515, 496, 496, 505, 506, 17, 6, 0, 0, 0, 0, 0, 0, 516, 533, 491, 507, 526, 460, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 460, 519, 511, 488, 532, 502, 505, 522, 5, 0, 0, 0, 0, 0, 0, 0, 564, 524, 502, 535, 504, 504, 464, 525, 16, 11, 0, 0, 0, 0, 0, 0, 517, 471, 533, 502, 499, 562, 529, 453, 0, 0, 0, 0, 0, 0, 0, 0, 541, 496, 519, 485, 526, 509, 487, 512, 27, 0, 0, 0, 0, 0, 0, 0, 507, 488, 499, 543, 515, 517, 506, 528, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4072, 4078, 4044, 4149, 4066, 4102, 4128] avg=4096.0000 max=4149 max_rank=4 min=4044 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 470, 511, 538, 556, 507, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 463, 504, 499, 534, 523, 523, 494, 487, 14, 4, 0, 0, 0, 0, 0, 0, 526, 486, 501, 533, 486, 455, 474, 528, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 498, 551, 521, 508, 515, 486, 3, 0, 0, 0, 0, 0, 0, 0, 505, 502, 505, 495, 494, 520, 462, 529, 13, 7, 0, 0, 0, 0, 0, 0, 505, 508, 535, 516, 533, 512, 533, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 553, 503, 506, 527, 524, 520, 21, 0, 0, 0, 0, 0, 0, 0, 493, 501, 506, 472, 528, 514, 535, 527, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4045, 3989, 4134, 4032, 4157, 4165, 4097] avg=4096.0000 max=4165 max_rank=6 min=3989 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[541, 513, 487, 493, 507, 518, 534, 527, 516, 501, 504, 483, 506, 515, 508, 516, 482, 500, 512, 549, 543, 542, 506, 529, 475, 487, 556, 531, 521, 496, 509, 516, 540, 524, 499, 461, 476, 546, 543, 539, 456, 549, 512, 505, 542, 485, 532, 489, 508, 516, 510, 502, 556, 514, 483, 503, 524, 511, 498, 519, 508, 471, 515, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[493, 520, 459, 491, 514, 487, 531, 532, 523, 546, 547, 460, 489, 514, 534, 561, 551, 533, 513, 530, 486, 504, 510, 516, 521, 533, 536, 536, 489, 491, 504, 516, 508, 518, 498, 518, 497, 489, 528, 524, 498, 487, 494, 517, 499, 523, 522, 508, 498, 489, 460, 516, 492, 514, 514, 488, 566, 519, 508, 506, 524, 555, 506, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[512, 504, 528, 556, 502, 496, 512, 519, 530, 534, 490, 509, 502, 477, 484, 527, 523, 511, 515, 521, 528, 532, 516, 521, 514, 453, 548, 492, 453, 499, 503, 548, 558, 516, 515, 498, 526, 490, 515, 503, 500, 543, 538, 532, 542, 517, 524, 520, 545, 496, 531, 474, 513, 508, 492, 530, 511, 516, 486, 485, 511, 511, 489, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[516, 503, 502, 529, 486, 499, 520, 491, 520, 499, 514, 526, 535, 518, 523, 505, 491, 498, 528, 528, 516, 519, 520, 468, 501, 509, 540, 487, 480, 546, 511, 524, 523, 536, 515, 459, 517, 490, 496, 513, 517, 428, 496, 506, 509, 545, 496, 529, 517, 500, 539, 502, 513, 529, 523, 550, 556, 516, 505, 504, 541, 504, 520, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[499, 499, 553, 505, 560, 479, 495, 521, 523, 549, 505, 523, 520, 534, 486, 482, 479, 498, 509, 505, 525, 519, 545, 510, 503, 517, 501, 511, 506, 533, 537, 502, 476, 496, 489, 546, 471, 465, 504, 514, 473, 472, 525, 548, 470, 552, 483, 511, 506, 487, 521, 498, 539, 556, 524, 515, 504, 493, 538, 522, 525, 522, 537, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[518, 516, 525, 525, 475, 536, 519, 481, 516, 491, 552, 507, 478, 525, 527, 532, 500, 519, 520, 504, 505, 540, 508, 547, 516, 511, 526, 511, 507, 504, 490, 520, 471, 483, 544, 515, 502, 472, 531, 491, 503, 534, 509, 526, 489, 499, 545, 511, 546, 526, 515, 498, 485, 492, 509, 512, 530, 502, 480, 515, 488, 522, 536, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[467, 467, 535, 476, 536, 489, 511, 498, 513, 518, 537, 513, 506, 527, 524, 529, 527, 539, 539, 505, 503, 498, 480, 521, 480, 500, 498, 477, 505, 517, 509, 492, 489, 540, 495, 507, 513, 500, 482, 515, 492, 525, 512, 534, 527, 527, 515, 503, 520, 557, 508, 518, 525, 534, 491, 571, 530, 508, 508, 525, 517, 525, 493, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[520, 505, 536, 516, 486, 535, 522, 483, 536, 519, 502, 496, 486, 511, 526, 497, 546, 453, 526, 528, 542, 497, 526, 492, 526, 516, 471, 492, 526, 551, 497, 519, 482, 510, 482, 527, 518, 553, 521, 543, 480, 529, 492, 511, 532, 522, 502, 513, 532, 500, 495, 487, 502, 502, 552, 516, 531, 500, 539, 518, 484, 486, 512, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 459, 491, 514, 487, 531, 532, 13, 6, 0, 0, 0, 0, 0, 0, 506, 546, 547, 460, 489, 514, 534, 561, 0, 0, 0, 0, 0, 0, 0, 0, 551, 533, 513, 502, 486, 504, 510, 516, 0, 0, 0, 0, 0, 0, 0, 0, 521, 533, 536, 536, 489, 491, 504, 516, 18, 4, 0, 0, 0, 0, 0, 0, 508, 518, 498, 518, 497, 489, 528, 524, 28, 1, 0, 0, 0, 0, 0, 0, 498, 487, 494, 517, 499, 523, 522, 508, 4, 0, 0, 0, 0, 0, 0, 0, 498, 489, 460, 516, 492, 514, 514, 478, 0, 0, 0, 0, 0, 0, 0, 0, 547, 519, 508, 506, 524, 555, 506, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4157, 4115, 4148, 4109, 4052, 3961, 4180] avg=4096.0000 max=4180 max_rank=7 min=3961 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[541, 513, 487, 493, 507, 518, 534, 527, 12, 6, 0, 0, 0, 0, 0, 0, 500, 501, 504, 483, 506, 515, 508, 516, 0, 0, 0, 0, 0, 0, 0, 0, 482, 500, 512, 519, 543, 542, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 475, 487, 556, 531, 521, 496, 509, 516, 17, 4, 0, 0, 0, 0, 0, 0, 540, 524, 499, 461, 476, 546, 543, 539, 30, 1, 0, 0, 0, 0, 0, 0, 456, 549, 512, 505, 542, 485, 532, 489, 5, 0, 0, 0, 0, 0, 0, 0, 508, 516, 510, 502, 556, 514, 483, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 498, 519, 508, 471, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4033, 4133, 4112, 4159, 4075, 4081, 4037] avg=4096.0000 max=4159 max_rank=4 min=4033 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 499, 553, 505, 560, 479, 495, 521, 13, 6, 0, 0, 0, 0, 0, 0, 506, 549, 505, 523, 520, 534, 486, 482, 0, 0, 0, 0, 0, 0, 0, 0, 479, 498, 509, 478, 525, 519, 545, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 501, 511, 506, 533, 537, 502, 16, 4, 0, 0, 0, 0, 0, 0, 476, 496, 489, 546, 471, 465, 504, 514, 27, 1, 0, 0, 0, 0, 0, 0, 473, 472, 525, 548, 470, 552, 483, 511, 5, 0, 0, 0, 0, 0, 0, 0, 506, 487, 521, 498, 539, 556, 524, 504, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 538, 522, 525, 522, 537, 553, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4105, 4063, 4130, 3989, 4039, 4135, 4177] avg=4096.0000 max=4177 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 504, 528, 556, 502, 496, 512, 519, 13, 6, 0, 0, 0, 0, 0, 0, 513, 534, 490, 509, 502, 477, 484, 527, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 515, 493, 528, 532, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 514, 453, 548, 492, 453, 499, 503, 548, 16, 4, 0, 0, 0, 0, 0, 0, 558, 516, 515, 498, 526, 490, 515, 503, 28, 1, 0, 0, 0, 0, 0, 0, 500, 543, 538, 532, 542, 517, 524, 520, 5, 0, 0, 0, 0, 0, 0, 0, 545, 496, 531, 474, 513, 508, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 494, 516, 486, 485, 511, 511, 489, 474, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4036, 4139, 4030, 4150, 4221, 4078, 3966] avg=4096.0000 max=4221 max_rank=5 min=3966 min_rank=7 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 516, 525, 525, 475, 536, 519, 481, 12, 6, 0, 0, 0, 0, 0, 0, 500, 491, 552, 507, 478, 525, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 520, 477, 505, 540, 508, 547, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 526, 511, 507, 504, 490, 520, 17, 4, 0, 0, 0, 0, 0, 0, 471, 483, 544, 515, 502, 472, 531, 491, 27, 1, 0, 0, 0, 0, 0, 0, 503, 534, 509, 526, 489, 499, 545, 511, 5, 0, 0, 0, 0, 0, 0, 0, 546, 526, 515, 498, 485, 492, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 502, 480, 515, 488, 522, 536, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4112, 4116, 4106, 4037, 4121, 4072, 4091] avg=4096.0000 max=4121 max_rank=5 min=4037 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 503, 502, 529, 486, 499, 520, 491, 12, 7, 0, 0, 0, 0, 0, 0, 504, 499, 514, 526, 535, 518, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 528, 500, 516, 519, 520, 468, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 540, 487, 480, 546, 511, 524, 18, 4, 0, 0, 0, 0, 0, 0, 523, 536, 515, 459, 517, 490, 496, 513, 28, 1, 0, 0, 0, 0, 0, 0, 517, 428, 496, 506, 509, 545, 496, 529, 5, 0, 0, 0, 0, 0, 0, 0, 517, 500, 539, 502, 513, 529, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 505, 504, 541, 504, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4124, 4040, 4120, 4078, 4031, 4161, 4149] avg=4096.0000 max=4161 max_rank=6 min=4031 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[467, 467, 535, 476, 536, 489, 511, 498, 16, 11, 0, 0, 0, 0, 0, 0, 491, 518, 537, 513, 506, 527, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 527, 539, 539, 473, 503, 498, 480, 521, 0, 0, 0, 0, 0, 0, 0, 0, 480, 500, 498, 477, 505, 517, 509, 492, 19, 6, 0, 0, 0, 0, 0, 0, 489, 540, 495, 507, 513, 500, 482, 515, 32, 8, 0, 0, 0, 0, 0, 0, 492, 525, 512, 534, 527, 527, 515, 503, 8, 0, 0, 0, 0, 0, 0, 0, 520, 557, 508, 518, 525, 534, 491, 552, 0, 0, 0, 0, 0, 0, 0, 0, 503, 508, 508, 525, 517, 525, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4145, 4080, 4003, 4081, 4143, 4205, 4105] avg=4096.0000 max=4205 max_rank=6 min=4003 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[520, 505, 536, 516, 486, 535, 522, 483, 13, 6, 0, 0, 0, 0, 0, 0, 519, 519, 502, 496, 486, 511, 526, 497, 0, 0, 0, 0, 0, 0, 0, 0, 546, 453, 526, 500, 542, 497, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 471, 492, 526, 551, 497, 519, 17, 4, 0, 0, 0, 0, 0, 0, 482, 510, 482, 527, 518, 553, 521, 543, 28, 1, 0, 0, 0, 0, 0, 0, 480, 529, 492, 511, 532, 522, 502, 513, 5, 0, 0, 0, 0, 0, 0, 0, 532, 500, 495, 487, 502, 502, 552, 505, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 539, 518, 484, 486, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4056, 4082, 4119, 4165, 4086, 4075, 4063] avg=4096.0000 max=4165 max_rank=4 min=4056 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[515, 517, 548, 504, 495, 502, 499, 540, 521, 522, 535, 495, 483, 520, 529, 514, 516, 490, 529, 483, 491, 502, 489, 505, 497, 519, 522, 528, 491, 518, 488, 504, 499, 517, 526, 512, 495, 531, 534, 534, 539, 479, 510, 498, 500, 522, 456, 499, 524, 512, 524, 498, 503, 507, 513, 513, 529, 496, 492, 540, 540, 561, 497, 557] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[526, 499, 496, 485, 538, 494, 544, 490, 506, 510, 544, 519, 473, 461, 509, 541, 462, 493, 541, 488, 520, 569, 521, 492, 509, 473, 537, 561, 504, 475, 506, 543, 550, 514, 525, 526, 476, 524, 549, 496, 515, 522, 522, 511, 517, 511, 503, 526, 525, 486, 524, 504, 511, 503, 540, 497, 536, 488, 481, 492, 479, 545, 513, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[547, 542, 478, 506, 537, 537, 505, 540, 506, 556, 527, 512, 522, 500, 504, 491, 541, 500, 528, 498, 494, 496, 494, 535, 538, 546, 509, 507, 555, 489, 495, 517, 466, 489, 493, 534, 515, 526, 525, 486, 535, 513, 521, 467, 528, 517, 505, 480, 531, 543, 509, 514, 514, 516, 500, 509, 493, 530, 467, 493, 567, 493, 447, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[515, 511, 498, 482, 506, 470, 493, 543, 528, 543, 511, 511, 529, 505, 511, 533, 498, 515, 532, 473, 517, 519, 515, 523, 498, 511, 518, 466, 512, 516, 510, 549, 479, 542, 492, 495, 555, 491, 504, 528, 524, 552, 563, 532, 478, 501, 523, 498, 483, 501, 530, 502, 522, 474, 470, 509, 502, 525, 494, 533, 510, 532, 533, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[488, 507, 498, 488, 501, 546, 536, 525, 490, 508, 490, 522, 545, 472, 538, 502, 551, 531, 495, 558, 522, 446, 484, 559, 500, 494, 519, 521, 478, 488, 472, 472, 486, 536, 496, 523, 520, 533, 502, 544, 538, 502, 534, 496, 545, 504, 513, 518, 512, 539, 517, 551, 512, 524, 519, 503, 518, 503, 490, 469, 544, 481, 540, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[503, 535, 532, 540, 489, 542, 488, 551, 539, 504, 546, 475, 474, 520, 549, 493, 538, 463, 515, 536, 513, 532, 509, 518, 487, 477, 497, 530, 488, 547, 510, 514, 526, 514, 472, 522, 514, 513, 517, 495, 496, 508, 553, 457, 522, 520, 519, 498, 526, 536, 551, 487, 484, 462, 505, 541, 533, 533, 461, 506, 517, 519, 504, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[511, 479, 518, 506, 538, 480, 506, 513, 536, 514, 509, 546, 537, 516, 515, 506, 502, 550, 501, 567, 511, 511, 517, 476, 536, 491, 496, 545, 509, 539, 472, 541, 516, 542, 498, 509, 541, 464, 513, 553, 509, 490, 495, 528, 500, 496, 514, 511, 507, 497, 496, 502, 467, 495, 487, 528, 531, 537, 542, 511, 472, 480, 508, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[521, 526, 500, 494, 485, 493, 536, 509, 476, 533, 542, 490, 506, 552, 505, 550, 485, 508, 491, 514, 556, 505, 493, 524, 487, 520, 504, 510, 496, 532, 457, 496, 475, 483, 515, 496, 555, 525, 511, 532, 494, 510, 554, 495, 537, 470, 463, 501, 537, 527, 542, 506, 513, 555, 504, 525, 525, 525, 487, 534, 514, 527, 510, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[488, 507, 498, 488, 501, 546, 536, 523, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 467, 522, 545, 472, 538, 502, 0, 0, 0, 0, 0, 0, 0, 0, 551, 531, 495, 558, 522, 446, 484, 559, 2, 0, 0, 0, 0, 0, 0, 0, 500, 494, 519, 521, 478, 488, 472, 472, 23, 3, 0, 0, 0, 0, 0, 0, 486, 536, 496, 523, 509, 533, 502, 544, 0, 0, 0, 0, 0, 0, 0, 0, 538, 502, 534, 496, 545, 504, 513, 518, 8, 4, 0, 0, 0, 0, 0, 0, 512, 539, 517, 551, 512, 524, 519, 503, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 490, 469, 544, 481, 540, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4044, 4148, 3970, 4129, 4162, 4177, 4051] avg=4096.0000 max=4177 max_rank=6 min=3970 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16519 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[526, 499, 496, 485, 538, 494, 544, 488, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 518, 519, 473, 461, 509, 541, 0, 0, 0, 0, 0, 0, 0, 0, 462, 493, 541, 488, 520, 569, 521, 492, 2, 0, 0, 0, 0, 0, 0, 0, 509, 473, 537, 561, 504, 475, 506, 543, 26, 3, 0, 0, 0, 0, 0, 0, 550, 514, 525, 526, 465, 524, 549, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 522, 511, 517, 511, 503, 526, 8, 4, 0, 0, 0, 0, 0, 0, 525, 486, 524, 504, 511, 503, 540, 497, 0, 0, 0, 0, 0, 0, 0, 0, 536, 488, 481, 492, 479, 545, 513, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4037, 4088, 4137, 4149, 4139, 4090, 4058] avg=4096.0000 max=4149 max_rank=4 min=4037 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16436 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 517, 548, 504, 495, 502, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 521, 522, 510, 495, 483, 520, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 516, 490, 529, 483, 491, 502, 489, 505, 2, 0, 0, 0, 0, 0, 0, 0, 497, 519, 522, 528, 491, 518, 488, 504, 25, 3, 0, 0, 0, 0, 0, 0, 499, 517, 526, 512, 484, 531, 534, 534, 0, 0, 0, 0, 0, 0, 0, 0, 539, 479, 510, 498, 500, 522, 456, 499, 8, 5, 0, 0, 0, 0, 0, 0, 524, 512, 524, 498, 503, 507, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 492, 540, 540, 561, 497, 552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4094, 4007, 4095, 4137, 4016, 4094, 4207] avg=4096.0000 max=4207 max_rank=7 min=4007 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16454 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[547, 542, 478, 506, 537, 537, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 506, 556, 502, 512, 522, 500, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 541, 500, 528, 498, 494, 496, 494, 535, 2, 0, 0, 0, 0, 0, 0, 0, 538, 546, 509, 507, 555, 489, 495, 517, 25, 3, 0, 0, 0, 0, 0, 0, 466, 489, 493, 534, 504, 526, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 535, 513, 521, 467, 528, 517, 505, 480, 8, 4, 0, 0, 0, 0, 0, 0, 531, 543, 509, 514, 514, 516, 500, 509, 0, 0, 0, 0, 0, 0, 0, 0, 493, 530, 467, 493, 567, 493, 447, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4093, 4088, 4184, 4023, 4078, 4136, 3976] avg=4096.0000 max=4190 max_rank=0 min=3976 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16213 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 511, 498, 482, 506, 470, 493, 541, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 487, 511, 529, 505, 511, 533, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 532, 473, 517, 519, 515, 523, 2, 0, 0, 0, 0, 0, 0, 0, 498, 511, 518, 466, 512, 516, 510, 549, 24, 3, 0, 0, 0, 0, 0, 0, 479, 542, 492, 495, 543, 491, 504, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 552, 563, 532, 478, 501, 523, 498, 9, 4, 0, 0, 0, 0, 0, 0, 483, 501, 530, 502, 522, 474, 470, 509, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 494, 533, 510, 532, 533, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4147, 4094, 4107, 4074, 4184, 3991, 4155] avg=4096.0000 max=4184 max_rank=5 min=3991 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16404 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 479, 518, 506, 538, 480, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 485, 546, 537, 516, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 502, 550, 501, 567, 511, 511, 517, 476, 2, 0, 0, 0, 0, 0, 0, 0, 536, 491, 496, 545, 509, 539, 472, 541, 24, 3, 0, 0, 0, 0, 0, 0, 516, 542, 498, 509, 529, 464, 513, 553, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 495, 528, 500, 496, 514, 511, 9, 4, 0, 0, 0, 0, 0, 0, 507, 497, 496, 502, 467, 495, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 531, 537, 542, 511, 472, 480, 508, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4155, 4137, 4156, 4124, 4056, 3979, 4112] avg=4096.0000 max=4156 max_rank=3 min=3979 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16271 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 535, 532, 540, 489, 542, 488, 549, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 520, 475, 474, 520, 549, 493, 0, 0, 0, 0, 0, 0, 0, 0, 538, 463, 515, 536, 513, 532, 509, 518, 2, 0, 0, 0, 0, 0, 0, 0, 487, 477, 497, 530, 488, 547, 510, 514, 26, 3, 0, 0, 0, 0, 0, 0, 526, 514, 472, 522, 503, 513, 517, 495, 0, 0, 0, 0, 0, 0, 0, 0, 496, 508, 553, 457, 522, 520, 519, 498, 8, 4, 0, 0, 0, 0, 0, 0, 526, 536, 551, 487, 484, 462, 505, 541, 0, 0, 0, 0, 0, 0, 0, 0, 533, 533, 461, 506, 517, 519, 504, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4074, 4126, 4079, 4062, 4085, 4092, 4072] avg=4096.0000 max=4178 max_rank=0 min=4062 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16311 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[521, 526, 500, 494, 485, 493, 536, 505, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 512, 490, 506, 552, 505, 550, 0, 0, 0, 0, 0, 0, 0, 0, 485, 508, 491, 514, 556, 505, 493, 524, 4, 0, 0, 0, 0, 0, 0, 0, 487, 520, 504, 510, 496, 532, 457, 496, 30, 8, 0, 0, 0, 0, 0, 0, 475, 483, 515, 496, 533, 525, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 554, 495, 537, 470, 463, 501, 14, 10, 0, 0, 0, 0, 0, 0, 537, 527, 542, 506, 513, 555, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 487, 534, 514, 527, 510, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4124, 4080, 4040, 4070, 4048, 4209, 4137] avg=4096.0000 max=4209 max_rank=6 min=4040 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16464 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[479, 533, 500, 536, 483, 522, 518, 477, 465, 494, 508, 515, 530, 528, 532, 494, 520, 507, 485, 492, 508, 528, 464, 488, 461, 494, 508, 537, 495, 533, 516, 507, 550, 530, 501, 493, 486, 547, 544, 529, 525, 545, 515, 580, 490, 530, 526, 513, 498, 497, 511, 513, 531, 497, 573, 511, 488, 486, 531, 493, 528, 484, 526, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[487, 522, 545, 559, 539, 474, 517, 510, 495, 540, 548, 492, 508, 531, 511, 496, 511, 476, 527, 495, 526, 506, 477, 461, 491, 483, 525, 474, 531, 497, 506, 520, 516, 517, 515, 536, 532, 482, 509, 519, 529, 543, 492, 536, 537, 518, 494, 515, 481, 505, 540, 525, 535, 527, 535, 527, 492, 519, 479, 525, 502, 496, 502, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[509, 535, 536, 487, 517, 510, 468, 532, 517, 543, 507, 499, 469, 537, 534, 502, 505, 539, 482, 474, 514, 553, 518, 519, 505, 538, 513, 481, 496, 497, 547, 533, 524, 517, 529, 504, 511, 501, 518, 486, 501, 499, 529, 501, 497, 521, 514, 531, 555, 517, 497, 482, 508, 498, 503, 515, 505, 513, 523, 496, 518, 532, 509, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[512, 506, 509, 495, 483, 499, 498, 508, 500, 476, 507, 542, 526, 492, 510, 534, 508, 524, 497, 512, 500, 499, 505, 523, 503, 493, 564, 535, 529, 537, 501, 527, 530, 486, 528, 514, 522, 509, 510, 539, 521, 490, 511, 473, 539, 523, 525, 502, 486, 562, 487, 541, 514, 505, 523, 512, 509, 499, 497, 515, 511, 557, 455, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[507, 469, 494, 538, 490, 509, 509, 505, 500, 559, 501, 488, 491, 481, 501, 502, 531, 485, 487, 485, 521, 512, 509, 503, 524, 506, 512, 536, 534, 494, 525, 505, 519, 542, 526, 530, 525, 539, 507, 518, 518, 530, 525, 489, 503, 503, 481, 540, 527, 529, 519, 521, 505, 517, 480, 505, 553, 547, 495, 543, 525, 504, 497, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[504, 511, 513, 532, 499, 529, 496, 495, 510, 492, 501, 483, 535, 514, 511, 481, 520, 554, 496, 488, 541, 465, 477, 518, 535, 494, 474, 525, 566, 520, 514, 536, 506, 504, 513, 479, 489, 537, 543, 515, 592, 503, 481, 532, 507, 533, 513, 525, 487, 492, 534, 528, 517, 535, 503, 486, 522, 485, 521, 512, 498, 499, 522, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[511, 530, 515, 512, 496, 516, 519, 497, 513, 472, 499, 526, 501, 521, 518, 498, 572, 495, 472, 474, 498, 544, 531, 516, 518, 488, 547, 490, 503, 514, 501, 515, 522, 531, 497, 499, 480, 512, 500, 523, 529, 543, 521, 494, 518, 480, 523, 479, 537, 516, 537, 501, 473, 518, 502, 520, 501, 531, 531, 541, 525, 516, 517, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[540, 527, 540, 488, 537, 534, 499, 499, 503, 510, 505, 519, 519, 537, 508, 544, 501, 547, 520, 516, 525, 498, 546, 511, 524, 484, 494, 497, 490, 522, 524, 516, 508, 517, 536, 507, 476, 508, 481, 494, 510, 525, 498, 512, 518, 506, 507, 533, 499, 486, 488, 476, 493, 533, 558, 493, 522, 510, 518, 519, 512, 506, 496, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[487, 522, 545, 559, 539, 474, 517, 510, 7, 4, 0, 0, 0, 0, 0, 0, 495, 540, 548, 492, 508, 531, 511, 496, 17, 0, 0, 0, 0, 0, 0, 0, 511, 476, 527, 495, 526, 506, 477, 461, 33, 0, 0, 0, 0, 0, 0, 0, 491, 483, 525, 474, 531, 497, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 515, 536, 532, 482, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 496, 543, 492, 536, 537, 518, 494, 515, 0, 0, 0, 0, 0, 0, 0, 0, 481, 505, 540, 525, 535, 527, 528, 527, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 479, 525, 502, 496, 502, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4138, 4012, 4023, 4109, 4131, 4168, 4023] avg=4096.0000 max=4168 max_rank=6 min=4012 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[479, 533, 500, 536, 483, 522, 518, 477, 7, 4, 0, 0, 0, 0, 0, 0, 465, 494, 508, 515, 530, 528, 532, 494, 18, 0, 0, 0, 0, 0, 0, 0, 520, 507, 485, 492, 508, 528, 464, 488, 33, 0, 0, 0, 0, 0, 0, 0, 461, 494, 508, 537, 495, 533, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 532, 530, 501, 493, 486, 547, 544, 529, 0, 0, 0, 0, 0, 0, 0, 0, 492, 545, 515, 580, 490, 530, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 511, 513, 531, 497, 566, 511, 0, 0, 0, 0, 0, 0, 0, 0, 488, 486, 531, 493, 528, 484, 526, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4084, 4025, 4047, 4162, 4191, 4124, 4076] avg=4096.0000 max=4191 max_rank=5 min=4025 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 535, 536, 487, 517, 510, 468, 532, 6, 4, 0, 0, 0, 0, 0, 0, 517, 543, 507, 499, 469, 537, 534, 502, 17, 0, 0, 0, 0, 0, 0, 0, 505, 539, 482, 474, 514, 553, 518, 519, 31, 0, 0, 0, 0, 0, 0, 0, 505, 538, 513, 481, 496, 497, 547, 529, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 529, 504, 511, 501, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 470, 499, 529, 501, 497, 521, 514, 531, 0, 0, 0, 0, 0, 0, 0, 0, 555, 517, 497, 482, 508, 498, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 523, 496, 518, 532, 509, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4125, 4135, 4106, 4073, 4062, 4069, 4094] avg=4096.0000 max=4135 max_rank=2 min=4062 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 506, 509, 495, 483, 499, 498, 508, 7, 4, 0, 0, 0, 0, 0, 0, 500, 476, 507, 542, 526, 492, 510, 534, 18, 0, 0, 0, 0, 0, 0, 0, 508, 524, 497, 512, 500, 499, 505, 523, 32, 0, 0, 0, 0, 0, 0, 0, 503, 493, 564, 535, 529, 537, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 512, 486, 528, 514, 522, 509, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 490, 511, 473, 539, 523, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 486, 562, 487, 541, 514, 505, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 499, 497, 515, 511, 557, 455, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4105, 4100, 4185, 4120, 4052, 4123, 4062] avg=4096.0000 max=4185 max_rank=3 min=4021 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 469, 494, 538, 490, 509, 509, 505, 6, 4, 0, 0, 0, 0, 0, 0, 500, 559, 501, 488, 491, 481, 501, 502, 17, 0, 0, 0, 0, 0, 0, 0, 531, 485, 487, 485, 521, 512, 509, 503, 32, 0, 0, 0, 0, 0, 0, 0, 524, 506, 512, 536, 534, 494, 525, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 542, 526, 530, 525, 539, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 525, 489, 503, 503, 481, 540, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 519, 521, 505, 517, 474, 505, 0, 0, 0, 0, 0, 0, 0, 0, 553, 547, 495, 543, 525, 504, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4040, 4065, 4132, 4189, 4057, 4097, 4157] avg=4096.0000 max=4189 max_rank=4 min=4031 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 511, 513, 532, 499, 529, 496, 495, 6, 4, 0, 0, 0, 0, 0, 0, 510, 492, 501, 483, 535, 514, 511, 481, 17, 0, 0, 0, 0, 0, 0, 0, 520, 554, 496, 488, 541, 465, 477, 518, 37, 0, 0, 0, 0, 0, 0, 0, 535, 494, 474, 525, 566, 520, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 513, 479, 489, 537, 543, 515, 0, 0, 0, 0, 0, 0, 0, 0, 555, 503, 481, 532, 507, 533, 513, 525, 0, 0, 0, 0, 0, 0, 0, 0, 487, 492, 534, 528, 517, 535, 497, 486, 0, 0, 0, 0, 0, 0, 0, 0, 522, 485, 521, 512, 498, 499, 522, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4044, 4096, 4160, 4069, 4149, 4076, 4085] avg=4096.0000 max=4160 max_rank=3 min=4044 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 530, 515, 512, 496, 516, 519, 497, 12, 8, 7, 2, 0, 0, 0, 0, 513, 472, 499, 526, 501, 521, 518, 498, 21, 1, 0, 0, 0, 0, 0, 0, 572, 495, 472, 474, 498, 544, 531, 516, 37, 0, 0, 0, 0, 0, 0, 0, 518, 488, 547, 490, 503, 514, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 494, 531, 497, 499, 480, 512, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 491, 543, 521, 494, 518, 480, 523, 479, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 537, 501, 473, 518, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0, 501, 531, 531, 539, 525, 516, 517, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4070, 4139, 4068, 4036, 4049, 4092, 4189] avg=4096.0000 max=4189 max_rank=7 min=4036 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[540, 527, 540, 488, 537, 534, 499, 499, 7, 4, 0, 0, 0, 0, 0, 0, 503, 510, 505, 519, 519, 537, 508, 544, 17, 0, 0, 0, 0, 0, 0, 0, 501, 547, 520, 516, 525, 498, 546, 511, 32, 0, 0, 0, 0, 0, 0, 0, 524, 484, 494, 497, 490, 522, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 536, 507, 476, 508, 481, 494, 0, 0, 0, 0, 0, 0, 0, 0, 478, 525, 498, 512, 518, 506, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 499, 486, 488, 476, 493, 533, 551, 493, 0, 0, 0, 0, 0, 0, 0, 0, 522, 510, 518, 519, 512, 506, 496, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4162, 4196, 4047, 4010, 4077, 4019, 4082] avg=4096.0000 max=4196 max_rank=2 min=4010 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16188 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 3968, 4224, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[493, 532, 482, 509, 497, 496, 520, 544, 511, 541, 490, 491, 500, 522, 526, 514, 505, 514, 494, 528, 523, 509, 525, 520, 523, 503, 514, 506, 550, 498, 535, 492, 485, 514, 480, 520, 492, 518, 505, 502, 510, 520, 509, 539, 492, 533, 507, 515, 502, 532, 526, 497, 523, 513, 504, 525, 514, 497, 540, 500, 527, 500, 514, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[538, 510, 509, 529, 522, 529, 499, 501, 502, 515, 501, 522, 533, 516, 512, 538, 512, 527, 521, 524, 490, 470, 534, 531, 522, 525, 500, 538, 510, 517, 538, 497, 502, 556, 497, 528, 492, 544, 509, 556, 548, 479, 476, 481, 537, 494, 499, 531, 531, 466, 483, 527, 513, 520, 482, 513, 530, 482, 485, 513, 481, 502, 474, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[497, 525, 505, 525, 504, 504, 520, 531, 486, 543, 482, 509, 516, 507, 492, 520, 484, 477, 537, 526, 570, 508, 532, 535, 533, 537, 497, 520, 514, 486, 551, 492, 528, 515, 500, 495, 491, 483, 534, 545, 525, 523, 483, 532, 497, 531, 508, 533, 464, 515, 503, 491, 454, 538, 500, 506, 517, 545, 521, 477, 539, 499, 490, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[504, 532, 507, 478, 505, 485, 547, 534, 530, 482, 522, 485, 484, 525, 490, 514, 498, 536, 504, 514, 489, 515, 469, 549, 542, 517, 532, 541, 484, 527, 502, 522, 500, 517, 455, 500, 522, 538, 508, 486, 528, 535, 479, 530, 536, 560, 535, 533, 510, 537, 509, 487, 527, 500, 517, 479, 493, 497, 488, 504, 552, 486, 538, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[524, 544, 525, 514, 509, 523, 500, 523, 517, 507, 509, 518, 513, 527, 490, 517, 523, 490, 564, 498, 523, 544, 478, 506, 479, 492, 534, 541, 525, 543, 538, 494, 530, 455, 508, 501, 474, 549, 531, 536, 524, 527, 471, 527, 510, 505, 476, 497, 507, 526, 520, 536, 492, 488, 492, 505, 493, 510, 508, 481, 551, 478, 490, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[508, 499, 499, 519, 496, 469, 497, 525, 471, 485, 482, 494, 488, 510, 504, 501, 538, 524, 501, 528, 503, 562, 527, 483, 497, 512, 506, 517, 558, 530, 533, 539, 536, 518, 503, 529, 549, 523, 512, 525, 487, 525, 512, 489, 467, 508, 517, 521, 527, 535, 516, 490, 507, 492, 554, 516, 499, 505, 537, 502, 538, 511, 476, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[466, 508, 521, 520, 535, 529, 543, 534, 471, 524, 514, 524, 503, 526, 492, 473, 513, 499, 490, 535, 521, 515, 491, 555, 511, 509, 532, 511, 504, 471, 532, 486, 529, 518, 494, 473, 504, 507, 513, 534, 511, 504, 536, 549, 539, 514, 531, 514, 490, 460, 475, 515, 501, 525, 541, 505, 511, 513, 512, 527, 510, 559, 482, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[549, 537, 511, 499, 518, 534, 524, 505, 503, 543, 529, 496, 462, 519, 535, 533, 509, 518, 513, 522, 524, 499, 486, 527, 530, 507, 513, 502, 543, 515, 484, 489, 477, 492, 523, 529, 492, 515, 495, 493, 506, 492, 480, 476, 516, 508, 534, 504, 522, 499, 513, 561, 494, 511, 489, 484, 523, 514, 532, 536, 502, 536, 494, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 532, 482, 509, 497, 496, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 490, 491, 500, 522, 526, 514, 24, 8, 0, 0, 0, 0, 0, 0, 505, 514, 494, 528, 523, 509, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 514, 506, 550, 498, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 485, 514, 480, 520, 492, 518, 505, 502, 10, 0, 0, 0, 0, 0, 0, 0, 510, 520, 509, 539, 492, 527, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 502, 532, 526, 497, 523, 513, 504, 525, 34, 1, 0, 0, 0, 0, 0, 0, 514, 497, 540, 500, 527, 500, 514, 506, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4127, 4093, 4087, 4026, 4119, 4157, 4104] avg=4096.0000 max=4157 max_rank=6 min=4026 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16406 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[538, 510, 509, 529, 522, 529, 499, 483, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 501, 522, 533, 516, 512, 538, 25, 8, 0, 0, 0, 0, 0, 0, 512, 527, 521, 524, 490, 470, 534, 505, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 500, 538, 510, 517, 504, 497, 0, 0, 0, 0, 0, 0, 0, 0, 502, 556, 497, 528, 492, 544, 509, 556, 10, 0, 0, 0, 0, 0, 0, 0, 548, 479, 476, 481, 537, 489, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 466, 483, 527, 513, 520, 482, 513, 34, 1, 0, 0, 0, 0, 0, 0, 530, 482, 485, 513, 481, 502, 474, 505, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4172, 4083, 4113, 4194, 4040, 4070, 3977] avg=4096.0000 max=4194 max_rank=4 min=3977 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16281 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[497, 525, 505, 525, 504, 504, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 486, 543, 482, 509, 516, 507, 492, 520, 25, 8, 0, 0, 0, 0, 0, 0, 484, 477, 537, 526, 570, 508, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 533, 537, 497, 520, 514, 486, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 528, 515, 500, 495, 491, 483, 534, 545, 10, 0, 0, 0, 0, 0, 0, 0, 525, 523, 483, 532, 497, 525, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 464, 515, 503, 491, 454, 538, 500, 506, 35, 1, 0, 0, 0, 0, 0, 0, 517, 545, 521, 477, 539, 499, 490, 521, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4088, 4143, 4095, 4101, 4126, 4007, 4115] avg=4096.0000 max=4143 max_rank=2 min=4007 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16349 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[466, 508, 521, 520, 535, 529, 543, 516, 0, 0, 0, 0, 0, 0, 0, 0, 471, 524, 514, 524, 503, 526, 492, 473, 26, 8, 0, 0, 0, 0, 0, 0, 513, 499, 490, 535, 521, 515, 491, 528, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 532, 511, 504, 471, 498, 486, 0, 0, 0, 0, 0, 0, 0, 0, 529, 518, 494, 473, 504, 507, 513, 534, 10, 0, 0, 0, 0, 0, 0, 0, 511, 504, 536, 549, 539, 509, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 490, 460, 475, 515, 501, 525, 541, 505, 34, 1, 0, 0, 0, 0, 0, 0, 511, 513, 512, 527, 510, 559, 482, 514, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4061, 4092, 4022, 4082, 4193, 4047, 4133] avg=4096.0000 max=4193 max_rank=5 min=4022 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16455 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[504, 532, 507, 478, 505, 485, 547, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 482, 522, 485, 484, 525, 490, 514, 25, 8, 0, 0, 0, 0, 0, 0, 498, 536, 504, 514, 489, 515, 469, 523, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 532, 541, 484, 527, 470, 522, 0, 0, 0, 0, 0, 0, 0, 0, 500, 517, 455, 500, 522, 538, 508, 486, 10, 0, 0, 0, 0, 0, 0, 0, 528, 535, 479, 530, 536, 554, 535, 533, 0, 0, 0, 0, 0, 0, 0, 0, 510, 537, 509, 487, 527, 500, 517, 479, 32, 1, 0, 0, 0, 0, 0, 0, 493, 497, 488, 504, 552, 486, 538, 517, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4065, 4048, 4135, 4036, 4230, 4099, 4081] avg=4096.0000 max=4230 max_rank=5 min=4036 min_rank=4 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16446 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[524, 544, 525, 514, 509, 523, 500, 505, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 509, 518, 513, 527, 490, 517, 23, 8, 0, 0, 0, 0, 0, 0, 523, 490, 564, 498, 523, 544, 478, 482, 0, 0, 0, 0, 0, 0, 0, 0, 479, 492, 534, 541, 525, 543, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 530, 455, 508, 501, 474, 549, 531, 536, 10, 0, 0, 0, 0, 0, 0, 0, 524, 527, 471, 527, 510, 500, 476, 497, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 520, 536, 492, 488, 492, 505, 34, 1, 0, 0, 0, 0, 0, 0, 493, 510, 508, 481, 551, 478, 490, 538, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4129, 4102, 4112, 4094, 4032, 4101, 4054] avg=4096.0000 max=4144 max_rank=0 min=4032 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16281 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 499, 499, 519, 496, 469, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 471, 485, 482, 494, 488, 510, 504, 501, 22, 8, 0, 0, 0, 0, 0, 0, 538, 524, 501, 528, 503, 562, 527, 460, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 506, 517, 558, 530, 499, 539, 0, 0, 0, 0, 0, 0, 0, 0, 536, 518, 503, 529, 549, 523, 512, 525, 10, 0, 0, 0, 0, 0, 0, 0, 487, 525, 512, 489, 467, 503, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 516, 490, 507, 492, 554, 516, 34, 1, 0, 0, 0, 0, 0, 0, 499, 505, 537, 502, 538, 511, 476, 537, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 3965, 4143, 4158, 4205, 4021, 4172, 4110] avg=4096.0000 max=4205 max_rank=4 min=3965 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16508 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[549, 537, 511, 499, 518, 534, 524, 479, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 529, 496, 462, 519, 535, 533, 29, 12, 0, 0, 0, 0, 0, 0, 509, 518, 513, 522, 524, 499, 486, 493, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 513, 502, 543, 515, 447, 489, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 523, 529, 492, 515, 495, 493, 14, 0, 0, 0, 0, 0, 0, 0, 506, 492, 480, 476, 516, 499, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 522, 499, 513, 561, 494, 511, 489, 484, 37, 5, 0, 0, 0, 0, 0, 0, 523, 514, 532, 536, 502, 536, 494, 548, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4161, 4064, 4046, 4030, 4007, 4115, 4194] avg=4096.0000 max=4194 max_rank=7 min=4007 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16346 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[511, 507, 522, 522, 505, 530, 518, 492, 496, 489, 505, 518, 543, 504, 493, 537, 528, 519, 514, 545, 505, 489, 499, 483, 499, 537, 499, 496, 497, 510, 511, 491, 489, 506, 524, 510, 494, 512, 512, 472, 544, 546, 547, 483, 515, 560, 492, 489, 535, 531, 512, 517, 529, 477, 542, 539, 493, 491, 493, 507, 517, 527, 507, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[528, 544, 512, 508, 466, 505, 542, 518, 480, 490, 492, 489, 535, 529, 503, 529, 506, 499, 498, 494, 572, 516, 511, 520, 503, 532, 486, 516, 532, 520, 506, 480, 543, 494, 511, 508, 520, 495, 501, 502, 535, 519, 522, 493, 528, 524, 522, 501, 536, 548, 556, 495, 483, 504, 477, 529, 533, 501, 501, 484, 503, 539, 477, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[496, 485, 528, 531, 521, 509, 538, 534, 531, 510, 516, 541, 497, 515, 474, 509, 514, 537, 502, 514, 496, 474, 536, 535, 495, 572, 491, 524, 524, 482, 478, 504, 547, 484, 501, 524, 501, 451, 539, 531, 522, 554, 519, 501, 498, 538, 528, 532, 537, 514, 501, 513, 512, 467, 541, 544, 462, 487, 477, 471, 498, 518, 536, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[520, 508, 540, 496, 486, 516, 498, 486, 498, 521, 527, 503, 497, 462, 518, 507, 556, 494, 468, 487, 539, 504, 485, 502, 539, 521, 495, 514, 508, 519, 542, 483, 503, 505, 540, 529, 533, 554, 497, 536, 524, 488, 498, 513, 518, 509, 496, 490, 519, 501, 515, 511, 513, 511, 479, 525, 519, 525, 471, 589, 551, 485, 553, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[505, 483, 480, 507, 502, 509, 533, 519, 459, 517, 525, 520, 481, 522, 514, 497, 523, 515, 510, 509, 512, 517, 516, 521, 555, 493, 520, 501, 490, 497, 478, 525, 533, 480, 511, 513, 546, 504, 572, 511, 525, 487, 544, 531, 526, 482, 492, 543, 486, 515, 523, 525, 513, 506, 544, 532, 544, 490, 501, 542, 483, 514, 501, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[484, 512, 550, 544, 497, 515, 532, 499, 534, 481, 484, 481, 526, 490, 521, 517, 535, 507, 513, 523, 477, 540, 493, 474, 541, 521, 538, 533, 518, 503, 534, 523, 500, 553, 527, 483, 530, 524, 528, 529, 491, 523, 519, 501, 522, 512, 476, 498, 536, 468, 529, 516, 502, 542, 496, 498, 508, 510, 519, 519, 468, 526, 473, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[538, 510, 539, 503, 500, 480, 497, 513, 494, 487, 504, 545, 493, 555, 531, 533, 469, 510, 512, 533, 504, 500, 500, 513, 485, 483, 492, 521, 553, 509, 501, 497, 483, 507, 465, 487, 550, 538, 537, 526, 529, 547, 517, 503, 524, 541, 483, 537, 470, 556, 525, 509, 523, 470, 536, 507, 497, 472, 537, 547, 503, 513, 511, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[517, 512, 488, 531, 533, 476, 519, 536, 529, 499, 531, 522, 485, 520, 465, 510, 530, 539, 495, 516, 491, 490, 488, 480, 524, 484, 517, 555, 517, 516, 533, 532, 504, 498, 524, 501, 553, 519, 499, 492, 527, 523, 526, 528, 496, 500, 512, 517, 534, 543, 485, 508, 512, 506, 522, 534, 480, 482, 490, 484, 512, 521, 530, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[511, 507, 522, 522, 505, 530, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 496, 489, 505, 518, 543, 504, 493, 537, 27, 2, 0, 0, 0, 0, 0, 0, 528, 519, 514, 545, 505, 489, 499, 483, 14, 3, 2, 0, 0, 0, 0, 0, 499, 537, 499, 493, 497, 510, 511, 491, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 524, 510, 480, 512, 512, 472, 0, 0, 0, 0, 0, 0, 0, 0, 512, 546, 547, 483, 515, 560, 492, 489, 0, 0, 0, 0, 0, 0, 0, 0, 535, 531, 512, 517, 529, 477, 542, 512, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 493, 507, 517, 527, 507, 542, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4114, 4101, 4037, 4005, 4144, 4155, 4107] avg=4096.0000 max=4155 max_rank=6 min=4005 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 544, 512, 508, 466, 505, 540, 518, 0, 0, 0, 0, 0, 0, 0, 0, 480, 490, 492, 489, 535, 529, 503, 529, 27, 2, 0, 0, 0, 0, 0, 0, 506, 499, 498, 494, 572, 516, 511, 520, 15, 3, 2, 0, 0, 0, 0, 0, 503, 532, 486, 513, 532, 520, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 543, 494, 511, 508, 505, 495, 501, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 522, 493, 528, 524, 522, 501, 0, 0, 0, 0, 0, 0, 0, 0, 536, 548, 556, 495, 483, 504, 477, 502, 0, 0, 0, 0, 0, 0, 0, 0, 533, 501, 501, 484, 503, 539, 477, 523, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4076, 4136, 4072, 4059, 4112, 4101, 4091] avg=4096.0000 max=4136 max_rank=2 min=4059 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 483, 480, 507, 502, 509, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0, 459, 517, 525, 520, 481, 522, 514, 497, 27, 2, 0, 0, 0, 0, 0, 0, 523, 515, 510, 509, 512, 517, 516, 521, 16, 3, 2, 0, 0, 0, 0, 0, 555, 493, 520, 498, 490, 497, 478, 525, 0, 0, 0, 0, 0, 0, 0, 0, 533, 480, 511, 513, 530, 504, 572, 511, 0, 0, 0, 0, 0, 0, 0, 0, 494, 487, 544, 531, 526, 482, 492, 543, 0, 0, 0, 0, 0, 0, 0, 0, 486, 515, 523, 525, 513, 506, 544, 505, 0, 0, 0, 0, 0, 0, 0, 0, 544, 490, 501, 542, 483, 514, 501, 494, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4064, 4144, 4056, 4154, 4099, 4117, 4098] avg=4096.0000 max=4154 max_rank=4 min=4036 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[496, 485, 528, 531, 521, 509, 535, 534, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 516, 541, 497, 515, 474, 509, 31, 9, 0, 0, 0, 0, 0, 0, 514, 537, 502, 514, 496, 474, 536, 535, 19, 6, 3, 1, 0, 0, 0, 0, 495, 572, 491, 518, 524, 482, 478, 504, 0, 0, 0, 0, 0, 0, 0, 0, 547, 484, 501, 524, 482, 451, 539, 531, 0, 0, 0, 0, 0, 0, 0, 0, 479, 554, 519, 501, 498, 538, 528, 532, 0, 0, 0, 0, 0, 0, 0, 0, 537, 514, 501, 513, 512, 467, 541, 512, 0, 0, 0, 0, 0, 0, 0, 0, 462, 487, 477, 471, 498, 518, 536, 507, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4133, 4137, 4064, 4059, 4149, 4097, 3990] avg=4096.0000 max=4149 max_rank=5 min=3990 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 508, 540, 496, 486, 516, 496, 486, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 527, 503, 497, 462, 518, 507, 26, 2, 0, 0, 0, 0, 0, 0, 556, 494, 468, 487, 539, 504, 485, 502, 16, 3, 2, 0, 0, 0, 0, 0, 539, 521, 495, 511, 508, 519, 542, 483, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 540, 529, 517, 554, 497, 536, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 498, 513, 518, 509, 496, 490, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 515, 511, 513, 511, 479, 499, 0, 0, 0, 0, 0, 0, 0, 0, 519, 525, 471, 589, 551, 485, 553, 529, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4061, 4056, 4118, 4181, 4005, 4048, 4251] avg=4096.0000 max=4251 max_rank=7 min=4005 min_rank=5 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[484, 512, 550, 544, 497, 515, 530, 499, 0, 0, 0, 0, 0, 0, 0, 0, 534, 481, 484, 481, 526, 490, 521, 517, 25, 2, 0, 0, 0, 0, 0, 0, 535, 507, 513, 523, 477, 540, 493, 474, 15, 3, 2, 0, 0, 0, 0, 0, 541, 521, 538, 530, 518, 503, 534, 523, 0, 0, 0, 0, 0, 0, 0, 0, 500, 553, 527, 483, 515, 524, 528, 529, 0, 0, 0, 0, 0, 0, 0, 0, 462, 523, 519, 501, 522, 512, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 536, 468, 529, 516, 502, 542, 496, 473, 0, 0, 0, 0, 0, 0, 0, 0, 508, 510, 519, 519, 468, 526, 473, 502, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4061, 4082, 4208, 4159, 4013, 4062, 4052] avg=4096.0000 max=4208 max_rank=3 min=4013 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 512, 488, 531, 533, 476, 517, 536, 0, 0, 0, 0, 0, 0, 0, 0, 529, 499, 531, 522, 485, 520, 465, 510, 27, 2, 0, 0, 0, 0, 0, 0, 530, 539, 495, 516, 491, 490, 488, 480, 16, 3, 2, 0, 0, 0, 0, 0, 524, 484, 517, 552, 517, 516, 533, 532, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 524, 501, 537, 519, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 526, 528, 496, 500, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 534, 543, 485, 508, 512, 506, 522, 507, 0, 0, 0, 0, 0, 0, 0, 0, 480, 482, 490, 484, 512, 521, 530, 526, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4090, 4050, 4175, 4074, 4097, 4117, 4055] avg=4096.0000 max=4175 max_rank=3 min=4050 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[538, 510, 539, 503, 500, 480, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 487, 504, 545, 493, 555, 531, 533, 25, 2, 0, 0, 0, 0, 0, 0, 469, 510, 512, 533, 504, 500, 500, 513, 16, 3, 2, 0, 0, 0, 0, 0, 485, 483, 492, 518, 553, 509, 501, 497, 0, 0, 0, 0, 0, 0, 0, 0, 483, 507, 465, 487, 534, 538, 537, 526, 0, 0, 0, 0, 0, 0, 0, 0, 497, 547, 517, 503, 524, 541, 483, 537, 0, 0, 0, 0, 0, 0, 0, 0, 470, 556, 525, 509, 523, 470, 536, 482, 0, 0, 0, 0, 0, 0, 0, 0, 497, 472, 537, 547, 503, 513, 511, 514, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4169, 4062, 4038, 4077, 4149, 4071, 4124] avg=4096.0000 max=4169 max_rank=1 min=4038 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[482, 511, 497, 485, 508, 540, 508, 493, 519, 521, 507, 537, 518, 504, 500, 543, 517, 510, 510, 531, 509, 505, 514, 558, 522, 480, 497, 526, 481, 477, 494, 545, 473, 475, 525, 508, 504, 508, 523, 536, 516, 512, 489, 522, 531, 540, 547, 518, 478, 532, 468, 512, 535, 499, 544, 516, 526, 503, 495, 499, 505, 533, 529, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[522, 502, 519, 482, 480, 484, 528, 515, 505, 480, 491, 497, 548, 515, 525, 511, 496, 536, 545, 509, 508, 517, 531, 507, 497, 491, 495, 509, 497, 490, 485, 538, 535, 464, 499, 502, 481, 485, 500, 549, 543, 510, 504, 551, 469, 519, 510, 459, 543, 541, 516, 488, 525, 498, 559, 517, 510, 550, 542, 509, 551, 532, 522, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[510, 523, 532, 513, 528, 502, 523, 497, 515, 543, 537, 510, 534, 497, 502, 507, 513, 468, 554, 515, 507, 550, 512, 512, 529, 496, 532, 502, 483, 511, 495, 475, 547, 492, 484, 502, 488, 521, 485, 512, 537, 515, 512, 498, 533, 507, 523, 485, 520, 506, 508, 527, 534, 529, 486, 541, 485, 473, 513, 522, 522, 520, 509, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[511, 492, 514, 551, 502, 507, 471, 515, 505, 538, 499, 513, 481, 516, 508, 526, 519, 497, 503, 564, 531, 493, 522, 561, 513, 505, 511, 535, 494, 517, 504, 529, 495, 526, 565, 509, 500, 523, 498, 498, 528, 485, 494, 478, 491, 529, 531, 518, 519, 515, 512, 483, 488, 523, 515, 520, 505, 532, 509, 475, 498, 500, 547, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[507, 514, 509, 476, 555, 532, 530, 489, 528, 550, 517, 522, 553, 513, 515, 502, 493, 509, 548, 491, 501, 495, 525, 526, 512, 486, 509, 524, 514, 538, 534, 534, 498, 486, 520, 519, 510, 522, 513, 492, 496, 497, 529, 483, 507, 489, 483, 516, 501, 469, 530, 501, 535, 546, 536, 505, 515, 510, 509, 527, 483, 508, 482, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[509, 477, 487, 526, 482, 484, 528, 507, 496, 514, 545, 479, 492, 528, 509, 540, 489, 524, 481, 556, 523, 552, 507, 483, 479, 494, 506, 473, 524, 529, 537, 523, 471, 521, 520, 486, 519, 535, 558, 564, 517, 505, 501, 503, 519, 507, 515, 550, 512, 501, 537, 492, 514, 522, 485, 505, 511, 546, 532, 535, 496, 523, 470, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[471, 487, 517, 552, 519, 534, 536, 547, 536, 487, 542, 486, 508, 528, 529, 502, 499, 494, 494, 568, 477, 509, 515, 496, 521, 494, 525, 530, 498, 527, 535, 481, 517, 487, 501, 518, 462, 503, 497, 519, 511, 482, 500, 488, 517, 517, 530, 529, 579, 532, 529, 528, 518, 476, 484, 536, 515, 519, 496, 507, 543, 467, 503, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[471, 518, 493, 518, 530, 524, 527, 520, 505, 524, 523, 539, 490, 517, 477, 517, 521, 529, 542, 504, 524, 491, 516, 484, 480, 486, 525, 572, 534, 487, 506, 556, 531, 543, 525, 483, 523, 540, 489, 488, 528, 533, 476, 505, 531, 472, 491, 511, 521, 504, 488, 514, 514, 525, 428, 536, 533, 529, 493, 512, 493, 488, 539, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[482, 511, 497, 485, 508, 540, 508, 493, 14, 3, 0, 0, 0, 0, 0, 0, 519, 521, 484, 537, 518, 504, 500, 543, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 510, 504, 509, 505, 514, 558, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 497, 526, 481, 477, 494, 545, 9, 7, 0, 0, 0, 0, 0, 0, 473, 475, 525, 508, 504, 508, 523, 536, 24, 0, 0, 0, 0, 0, 0, 0, 516, 512, 489, 522, 531, 540, 547, 518, 8, 6, 0, 0, 0, 0, 0, 0, 478, 532, 468, 512, 535, 499, 544, 501, 0, 0, 0, 0, 0, 0, 0, 0, 526, 497, 495, 499, 505, 533, 529, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4126, 4127, 4038, 4076, 4189, 4069, 4102] avg=4096.0000 max=4189 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 519, 482, 480, 484, 528, 515, 14, 3, 0, 0, 0, 0, 0, 0, 505, 480, 468, 497, 548, 515, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 496, 536, 545, 483, 508, 517, 531, 507, 0, 0, 0, 0, 0, 0, 0, 0, 497, 491, 495, 509, 497, 490, 485, 538, 9, 7, 0, 0, 0, 0, 0, 0, 535, 464, 499, 502, 481, 485, 500, 549, 23, 0, 0, 0, 0, 0, 0, 0, 543, 510, 504, 551, 469, 519, 510, 459, 8, 7, 0, 0, 0, 0, 0, 0, 543, 541, 516, 488, 525, 498, 559, 502, 0, 0, 0, 0, 0, 0, 0, 0, 510, 543, 542, 509, 551, 532, 522, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4049, 4123, 4018, 4038, 4080, 4172, 4239] avg=4096.0000 max=4239 max_rank=7 min=4018 min_rank=3 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 523, 532, 513, 528, 502, 523, 497, 15, 3, 0, 0, 0, 0, 0, 0, 515, 543, 512, 510, 534, 497, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0, 513, 468, 554, 489, 507, 550, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 532, 502, 483, 511, 495, 475, 10, 8, 0, 0, 0, 0, 0, 0, 547, 492, 484, 502, 488, 521, 485, 512, 23, 0, 0, 0, 0, 0, 0, 0, 537, 515, 512, 498, 533, 507, 523, 485, 9, 6, 0, 0, 0, 0, 0, 0, 520, 506, 508, 527, 534, 529, 486, 524, 0, 0, 0, 0, 0, 0, 0, 0, 485, 467, 513, 522, 522, 520, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4120, 4105, 4041, 4054, 4125, 4134, 4043] avg=4096.0000 max=4146 max_rank=0 min=4041 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[471, 487, 517, 552, 519, 534, 536, 547, 15, 3, 0, 0, 0, 0, 0, 0, 536, 487, 517, 486, 508, 528, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 494, 539, 477, 509, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 525, 530, 498, 527, 535, 481, 10, 7, 0, 0, 0, 0, 0, 0, 517, 487, 501, 518, 462, 503, 497, 519, 26, 0, 0, 0, 0, 0, 0, 0, 511, 482, 500, 488, 517, 517, 530, 529, 8, 6, 0, 0, 0, 0, 0, 0, 579, 532, 529, 528, 518, 476, 484, 521, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 496, 507, 543, 467, 503, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4093, 4023, 4128, 4030, 4088, 4167, 4058] avg=4096.0000 max=4181 max_rank=0 min=4023 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 477, 487, 526, 482, 484, 528, 507, 15, 3, 0, 0, 0, 0, 0, 0, 496, 514, 520, 479, 492, 528, 509, 540, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 481, 528, 523, 552, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0, 479, 494, 506, 473, 524, 529, 537, 523, 10, 7, 0, 0, 0, 0, 0, 0, 471, 521, 520, 486, 519, 535, 558, 564, 25, 0, 0, 0, 0, 0, 0, 0, 517, 505, 501, 503, 519, 507, 515, 550, 8, 7, 0, 0, 0, 0, 0, 0, 512, 501, 537, 492, 514, 522, 485, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 539, 532, 535, 496, 523, 470, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4078, 4087, 4082, 4199, 4132, 4053, 4119] avg=4096.0000 max=4199 max_rank=4 min=4018 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 492, 514, 551, 502, 507, 471, 515, 14, 3, 0, 0, 0, 0, 0, 0, 505, 538, 476, 513, 481, 516, 508, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 503, 535, 531, 493, 522, 561, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 511, 535, 494, 517, 504, 529, 9, 7, 0, 0, 0, 0, 0, 0, 495, 526, 565, 509, 500, 523, 498, 498, 26, 0, 0, 0, 0, 0, 0, 0, 528, 485, 494, 478, 491, 529, 531, 518, 8, 7, 0, 0, 0, 0, 0, 0, 519, 515, 512, 483, 488, 523, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 509, 475, 498, 500, 547, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4063, 4161, 4124, 4140, 4069, 4060, 4071] avg=4096.0000 max=4161 max_rank=2 min=4060 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 514, 509, 476, 555, 532, 530, 489, 14, 3, 0, 0, 0, 0, 0, 0, 528, 550, 494, 522, 553, 513, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 509, 548, 466, 501, 495, 525, 526, 0, 0, 0, 0, 0, 0, 0, 0, 512, 486, 509, 524, 514, 538, 534, 534, 9, 7, 0, 0, 0, 0, 0, 0, 498, 486, 520, 519, 510, 522, 513, 492, 22, 0, 0, 0, 0, 0, 0, 0, 496, 497, 529, 483, 507, 489, 483, 516, 8, 6, 0, 0, 0, 0, 0, 0, 501, 469, 530, 501, 535, 546, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 509, 527, 483, 508, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4177, 4063, 4167, 4082, 4014, 4108, 4028] avg=4096.0000 max=4177 max_rank=1 min=4014 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[471, 518, 493, 518, 530, 524, 527, 520, 18, 5, 0, 0, 0, 0, 0, 0, 505, 524, 493, 539, 490, 517, 477, 517, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 542, 472, 524, 491, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0, 480, 486, 525, 572, 534, 487, 506, 556, 12, 12, 0, 0, 0, 0, 0, 0, 531, 543, 525, 483, 523, 540, 489, 488, 27, 0, 0, 0, 0, 0, 0, 0, 528, 533, 476, 505, 531, 472, 491, 511, 13, 11, 0, 0, 0, 0, 0, 0, 521, 504, 488, 514, 514, 525, 428, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 493, 512, 493, 488, 539, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4062, 4079, 4170, 4149, 4071, 4005, 4108] avg=4096.0000 max=4170 max_rank=3 min=4005 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 15:43:13] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 271.1 | throughput per GPU (TFLOP/s/GPU): 455.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.761441E+00 | loss scale: 1.0 | grad norm: 3.733 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[507, 525, 529, 521, 518, 495, 526, 513, 485, 536, 542, 509, 545, 484, 478, 536, 530, 495, 553, 478, 515, 511, 477, 500, 526, 491, 555, 512, 523, 502, 560, 521, 520, 463, 505, 480, 539, 519, 534, 486, 498, 514, 522, 508, 471, 524, 476, 496, 514, 495, 513, 489, 516, 538, 490, 504, 507, 515, 493, 501, 585, 537, 528, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[507, 515, 511, 543, 554, 494, 517, 468, 547, 526, 504, 500, 543, 508, 521, 525, 511, 529, 526, 517, 553, 530, 504, 479, 553, 506, 478, 496, 534, 479, 523, 492, 478, 458, 508, 504, 521, 535, 497, 516, 475, 530, 509, 483, 491, 493, 522, 535, 554, 488, 522, 524, 505, 486, 530, 550, 495, 500, 460, 502, 526, 544, 507, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[532, 533, 511, 525, 515, 509, 493, 485, 535, 524, 506, 496, 509, 542, 540, 550, 476, 523, 480, 540, 523, 497, 497, 522, 472, 522, 515, 498, 504, 532, 519, 521, 495, 535, 514, 539, 488, 494, 487, 489, 480, 544, 525, 516, 502, 495, 525, 509, 552, 527, 540, 524, 498, 478, 534, 505, 505, 508, 510, 479, 506, 502, 496, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[474, 507, 475, 541, 483, 508, 529, 515, 527, 489, 487, 522, 475, 532, 567, 500, 483, 517, 507, 482, 543, 506, 527, 538, 508, 542, 513, 533, 466, 517, 538, 528, 502, 503, 555, 532, 506, 524, 527, 503, 506, 500, 521, 528, 504, 482, 520, 496, 480, 513, 502, 524, 528, 508, 492, 528, 526, 545, 495, 519, 516, 494, 512, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[550, 482, 508, 508, 528, 484, 500, 523, 490, 540, 501, 531, 526, 511, 521, 520, 546, 527, 485, 505, 531, 503, 504, 479, 479, 513, 484, 528, 526, 485, 525, 543, 543, 503, 483, 512, 540, 513, 465, 502, 528, 475, 536, 460, 473, 485, 493, 492, 526, 510, 544, 520, 512, 531, 482, 543, 526, 526, 541, 533, 548, 516, 497, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[485, 513, 509, 516, 513, 501, 536, 500, 541, 517, 514, 530, 492, 547, 505, 498, 498, 525, 504, 529, 542, 532, 518, 538, 499, 552, 529, 504, 492, 532, 486, 494, 528, 505, 519, 532, 498, 475, 496, 509, 525, 518, 474, 500, 490, 449, 525, 519, 536, 495, 546, 493, 492, 519, 523, 512, 546, 527, 510, 529, 519, 484, 496, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[516, 501, 517, 475, 508, 499, 562, 499, 500, 508, 567, 507, 515, 479, 500, 503, 485, 526, 536, 522, 508, 507, 512, 522, 537, 534, 516, 505, 534, 512, 502, 525, 522, 542, 533, 495, 542, 455, 498, 512, 515, 488, 495, 507, 537, 485, 490, 520, 493, 523, 520, 478, 534, 513, 515, 509, 529, 504, 513, 500, 498, 544, 497, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[568, 518, 538, 521, 518, 503, 483, 476, 532, 474, 501, 510, 518, 524, 545, 521, 518, 545, 510, 493, 521, 541, 544, 497, 497, 505, 531, 541, 521, 526, 487, 490, 496, 513, 467, 528, 530, 490, 523, 548, 480, 491, 514, 508, 499, 549, 493, 513, 503, 517, 493, 475, 521, 512, 511, 509, 556, 514, 494, 489, 506, 500, 488, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 525, 529, 521, 518, 495, 526, 513, 4, 0, 0, 0, 0, 0, 0, 0, 485, 536, 542, 509, 545, 484, 443, 536, 0, 0, 0, 0, 0, 0, 0, 0, 530, 495, 553, 478, 497, 511, 477, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 475, 555, 512, 523, 502, 560, 521, 0, 0, 0, 0, 0, 0, 0, 0, 520, 463, 505, 480, 539, 519, 534, 486, 11, 7, 4, 0, 0, 0, 0, 0, 498, 514, 522, 508, 471, 524, 476, 496, 35, 18, 9, 0, 0, 0, 0, 0, 514, 495, 502, 489, 516, 538, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 493, 501, 577, 537, 528, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4080, 4041, 4174, 4068, 4071, 4048, 4148] avg=4096.0000 max=4174 max_rank=3 min=4041 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[474, 507, 475, 541, 483, 508, 529, 515, 3, 0, 0, 0, 0, 0, 0, 0, 527, 489, 487, 522, 475, 532, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 483, 517, 507, 482, 524, 506, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 513, 533, 466, 517, 538, 528, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 555, 532, 506, 524, 527, 503, 11, 8, 4, 0, 0, 0, 0, 0, 506, 500, 521, 528, 504, 482, 520, 496, 42, 19, 10, 0, 0, 0, 0, 0, 480, 513, 491, 524, 528, 508, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 545, 495, 519, 509, 494, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4057, 4084, 4127, 4175, 4128, 4064, 4098] avg=4096.0000 max=4175 max_rank=4 min=4035 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 533, 511, 525, 515, 509, 493, 485, 3, 0, 0, 0, 0, 0, 0, 0, 535, 524, 506, 496, 509, 542, 500, 550, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 480, 540, 504, 497, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 472, 505, 515, 498, 504, 532, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 495, 535, 514, 539, 488, 494, 487, 489, 12, 8, 4, 0, 0, 0, 0, 0, 480, 544, 525, 516, 502, 495, 525, 509, 40, 19, 9, 0, 0, 0, 0, 0, 552, 527, 528, 524, 498, 478, 534, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 510, 479, 499, 502, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4162, 4039, 4066, 4065, 4164, 4146, 4020] avg=4096.0000 max=4164 max_rank=5 min=4020 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 511, 543, 554, 494, 517, 468, 4, 0, 0, 0, 0, 0, 0, 0, 547, 526, 504, 500, 543, 508, 483, 525, 0, 0, 0, 0, 0, 0, 0, 0, 511, 529, 526, 517, 533, 530, 504, 479, 0, 0, 0, 0, 0, 0, 0, 0, 553, 489, 478, 496, 534, 479, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 478, 458, 508, 504, 521, 535, 497, 516, 11, 8, 4, 0, 0, 0, 0, 0, 475, 530, 509, 483, 491, 493, 522, 535, 38, 20, 9, 0, 0, 0, 0, 0, 554, 488, 511, 524, 505, 486, 530, 550, 0, 0, 0, 0, 0, 0, 0, 0, 495, 500, 460, 502, 518, 544, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4136, 4129, 4044, 4040, 4105, 4148, 4053] avg=4096.0000 max=4148 max_rank=6 min=4040 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 501, 517, 475, 508, 499, 562, 499, 3, 0, 0, 0, 0, 0, 0, 0, 500, 508, 567, 507, 515, 479, 463, 503, 0, 0, 0, 0, 0, 0, 0, 0, 485, 526, 536, 522, 490, 507, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 516, 505, 534, 512, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 533, 495, 542, 455, 498, 512, 11, 8, 4, 0, 0, 0, 0, 0, 515, 488, 495, 507, 537, 485, 490, 520, 37, 18, 10, 0, 0, 0, 0, 0, 493, 523, 509, 478, 534, 513, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 513, 500, 491, 544, 497, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4042, 4100, 4147, 4122, 4102, 4074, 4101] avg=4096.0000 max=4147 max_rank=3 min=4042 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[485, 513, 509, 516, 513, 501, 536, 500, 3, 0, 0, 0, 0, 0, 0, 0, 541, 517, 514, 530, 492, 547, 468, 498, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 504, 529, 523, 532, 518, 538, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 529, 504, 492, 532, 486, 494, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 519, 532, 498, 475, 496, 509, 12, 8, 4, 0, 0, 0, 0, 0, 525, 518, 474, 500, 490, 449, 525, 519, 37, 19, 10, 0, 0, 0, 0, 0, 536, 495, 534, 493, 492, 519, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 510, 529, 512, 484, 496, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4107, 4167, 4070, 4086, 4066, 4104, 4092] avg=4096.0000 max=4167 max_rank=2 min=4066 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[550, 482, 508, 508, 528, 484, 500, 523, 9, 0, 0, 0, 0, 0, 0, 0, 490, 540, 501, 531, 526, 511, 480, 520, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 485, 505, 508, 503, 504, 479, 0, 0, 0, 0, 0, 0, 0, 0, 479, 489, 484, 528, 526, 485, 525, 543, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 483, 512, 540, 513, 465, 502, 15, 12, 7, 0, 0, 0, 0, 0, 528, 475, 536, 460, 473, 485, 493, 492, 41, 23, 12, 0, 0, 0, 0, 0, 526, 510, 529, 520, 512, 531, 482, 543, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 541, 533, 532, 516, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4099, 4057, 4059, 4095, 4018, 4153, 4195] avg=4096.0000 max=4195 max_rank=7 min=4018 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[568, 518, 538, 521, 518, 503, 483, 476, 3, 0, 0, 0, 0, 0, 0, 0, 532, 474, 501, 510, 518, 524, 505, 521, 0, 0, 0, 0, 0, 0, 0, 0, 518, 545, 510, 493, 503, 541, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 497, 488, 531, 541, 521, 526, 487, 490, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 467, 528, 530, 490, 523, 548, 10, 8, 4, 0, 0, 0, 0, 0, 480, 491, 514, 508, 499, 549, 493, 513, 40, 18, 9, 0, 0, 0, 0, 0, 503, 517, 483, 475, 521, 512, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 556, 514, 494, 489, 499, 500, 488, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4085, 4151, 4081, 4117, 4114, 4031, 4061] avg=4096.0000 max=4151 max_rank=2 min=4031 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 3968, 4096, 4096, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[513, 535, 540, 498, 503, 504, 499, 538, 535, 503, 512, 509, 476, 520, 520, 517, 500, 528, 494, 499, 555, 491, 497, 522, 496, 486, 507, 482, 480, 514, 542, 479, 526, 518, 488, 517, 472, 539, 513, 529, 479, 499, 535, 490, 486, 530, 494, 475, 518, 513, 530, 533, 535, 512, 525, 495, 517, 565, 518, 537, 491, 539, 508, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[510, 494, 521, 504, 471, 527, 516, 496, 484, 496, 506, 512, 493, 531, 503, 537, 526, 479, 529, 516, 511, 486, 481, 526, 522, 506, 529, 499, 500, 524, 527, 530, 515, 494, 510, 510, 513, 486, 543, 525, 536, 472, 512, 518, 515, 512, 521, 535, 541, 477, 531, 518, 521, 513, 479, 519, 503, 529, 541, 496, 531, 526, 509, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[489, 503, 500, 519, 535, 525, 508, 530, 523, 497, 510, 513, 507, 497, 522, 513, 509, 529, 508, 564, 535, 510, 504, 498, 497, 507, 519, 525, 497, 532, 508, 511, 468, 504, 539, 501, 499, 470, 497, 519, 492, 500, 536, 508, 481, 497, 527, 500, 516, 493, 488, 506, 508, 528, 494, 522, 533, 493, 551, 530, 515, 541, 538, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[531, 466, 506, 522, 515, 486, 561, 505, 525, 548, 534, 537, 516, 539, 456, 538, 525, 473, 484, 524, 470, 534, 486, 517, 500, 511, 479, 500, 476, 547, 541, 504, 549, 470, 495, 515, 504, 518, 482, 497, 542, 544, 474, 480, 514, 528, 502, 555, 495, 524, 522, 517, 520, 523, 514, 558, 513, 515, 506, 523, 514, 511, 466, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[539, 511, 532, 498, 500, 498, 536, 510, 503, 488, 504, 543, 510, 506, 514, 511, 510, 504, 538, 546, 509, 543, 533, 495, 489, 516, 499, 504, 520, 525, 527, 517, 520, 528, 539, 483, 470, 546, 494, 521, 489, 482, 490, 506, 500, 516, 541, 509, 524, 513, 518, 524, 472, 535, 482, 495, 568, 487, 526, 522, 480, 490, 491, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[526, 514, 528, 506, 503, 485, 532, 494, 507, 495, 501, 507, 490, 539, 512, 499, 539, 546, 512, 501, 491, 516, 536, 502, 506, 525, 498, 535, 512, 470, 505, 502, 526, 521, 527, 505, 491, 522, 512, 520, 514, 495, 498, 498, 526, 471, 508, 478, 500, 560, 541, 542, 509, 493, 470, 543, 504, 500, 491, 530, 551, 542, 502, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[567, 501, 476, 523, 503, 517, 497, 526, 516, 515, 524, 532, 494, 563, 548, 498, 482, 480, 503, 540, 512, 514, 518, 508, 515, 499, 493, 505, 531, 498, 522, 490, 492, 513, 481, 520, 503, 526, 533, 518, 476, 517, 508, 493, 558, 531, 533, 539, 513, 518, 522, 506, 526, 503, 531, 509, 508, 506, 461, 526, 466, 514, 506, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[503, 529, 496, 539, 496, 516, 552, 522, 479, 484, 504, 530, 506, 491, 499, 484, 513, 542, 497, 516, 543, 480, 541, 513, 501, 521, 547, 528, 489, 495, 498, 502, 517, 496, 506, 504, 497, 530, 491, 531, 551, 542, 499, 533, 512, 537, 499, 481, 506, 495, 459, 454, 523, 508, 498, 498, 529, 511, 499, 545, 555, 529, 532, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[539, 511, 532, 498, 500, 498, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 488, 504, 543, 510, 506, 514, 511, 4, 1, 0, 0, 0, 0, 0, 0, 510, 504, 538, 538, 509, 543, 533, 495, 0, 0, 0, 0, 0, 0, 0, 0, 489, 516, 499, 504, 520, 525, 527, 517, 25, 0, 0, 0, 0, 0, 0, 0, 520, 528, 539, 483, 470, 546, 494, 521, 19, 0, 0, 0, 0, 0, 0, 0, 489, 482, 490, 506, 500, 516, 541, 509, 9, 4, 4, 0, 0, 0, 0, 0, 524, 513, 518, 524, 472, 535, 482, 494, 0, 0, 0, 0, 0, 0, 0, 0, 568, 487, 526, 522, 480, 490, 491, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4084, 4170, 4122, 4120, 4050, 4062, 4045] avg=4096.0000 max=4170 max_rank=2 min=4045 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 503, 500, 519, 535, 525, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 510, 513, 507, 497, 522, 513, 4, 1, 0, 0, 0, 0, 0, 0, 509, 529, 508, 556, 535, 510, 504, 498, 0, 0, 0, 0, 0, 0, 0, 0, 497, 507, 519, 525, 497, 532, 508, 511, 25, 0, 0, 0, 0, 0, 0, 0, 468, 504, 539, 501, 499, 470, 497, 519, 19, 0, 0, 0, 0, 0, 0, 0, 492, 500, 536, 508, 481, 497, 527, 500, 9, 4, 4, 0, 0, 0, 0, 0, 516, 493, 488, 506, 508, 528, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 551, 530, 515, 541, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4087, 4149, 4121, 4016, 4058, 4054, 4183] avg=4096.0000 max=4183 max_rank=7 min=4016 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 494, 521, 504, 471, 527, 504, 496, 0, 0, 0, 0, 0, 0, 0, 0, 484, 496, 506, 512, 493, 531, 503, 537, 7, 3, 0, 0, 0, 0, 0, 0, 526, 479, 529, 501, 511, 486, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 529, 499, 500, 524, 527, 530, 30, 0, 0, 0, 0, 0, 0, 0, 515, 494, 510, 510, 513, 486, 543, 525, 25, 0, 0, 0, 0, 0, 0, 0, 536, 472, 512, 518, 515, 512, 521, 535, 12, 9, 8, 0, 0, 0, 0, 0, 541, 477, 531, 518, 521, 513, 479, 516, 0, 0, 0, 0, 0, 0, 0, 0, 503, 529, 541, 496, 531, 526, 509, 461, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4072, 4039, 4167, 4121, 4150, 4096, 4096] avg=4096.0000 max=4167 max_rank=3 min=4027 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 535, 540, 498, 503, 504, 490, 538, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 512, 509, 476, 520, 520, 517, 3, 1, 0, 0, 0, 0, 0, 0, 500, 528, 494, 493, 555, 491, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 507, 482, 480, 514, 542, 479, 26, 0, 0, 0, 0, 0, 0, 0, 526, 518, 488, 517, 472, 539, 513, 529, 20, 0, 0, 0, 0, 0, 0, 0, 479, 499, 535, 490, 486, 530, 494, 475, 9, 5, 3, 0, 0, 0, 0, 0, 518, 513, 530, 533, 535, 512, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 517, 565, 518, 537, 491, 539, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4096, 4080, 4012, 4122, 4005, 4160, 4172] avg=4096.0000 max=4172 max_rank=7 min=4005 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 514, 528, 506, 503, 485, 523, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 501, 507, 490, 539, 512, 499, 3, 1, 0, 0, 0, 0, 0, 0, 539, 546, 512, 495, 491, 516, 536, 502, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 498, 535, 512, 470, 505, 502, 26, 0, 0, 0, 0, 0, 0, 0, 526, 521, 527, 505, 491, 522, 512, 520, 20, 0, 0, 0, 0, 0, 0, 0, 514, 495, 498, 498, 526, 471, 508, 478, 9, 5, 3, 0, 0, 0, 0, 0, 500, 560, 541, 542, 509, 493, 470, 542, 0, 0, 0, 0, 0, 0, 0, 0, 504, 500, 491, 530, 551, 542, 502, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4054, 4137, 4079, 4144, 4005, 4157, 4113] avg=4096.0000 max=4157 max_rank=6 min=4005 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 466, 506, 522, 515, 486, 551, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 548, 534, 537, 516, 539, 456, 538, 4, 1, 0, 0, 0, 0, 0, 0, 525, 473, 484, 517, 470, 534, 486, 517, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 479, 500, 476, 547, 541, 504, 25, 0, 0, 0, 0, 0, 0, 0, 549, 470, 495, 515, 504, 518, 482, 497, 19, 0, 0, 0, 0, 0, 0, 0, 542, 544, 474, 480, 514, 528, 502, 555, 10, 4, 3, 0, 0, 0, 0, 0, 495, 524, 522, 517, 520, 523, 514, 557, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 506, 523, 514, 511, 466, 474, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4198, 4006, 4083, 4049, 4156, 4172, 4022] avg=4096.0000 max=4198 max_rank=1 min=4006 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[503, 529, 496, 539, 496, 516, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 479, 484, 504, 530, 506, 491, 499, 484, 4, 1, 0, 0, 0, 0, 0, 0, 513, 542, 497, 509, 543, 480, 541, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 547, 528, 489, 495, 498, 502, 26, 0, 0, 0, 0, 0, 0, 0, 517, 496, 506, 504, 497, 530, 491, 531, 20, 0, 0, 0, 0, 0, 0, 0, 551, 542, 499, 533, 512, 537, 499, 481, 10, 5, 3, 0, 0, 0, 0, 0, 506, 495, 459, 454, 523, 508, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 499, 545, 555, 529, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 3982, 4138, 4107, 4092, 4172, 3940, 4194] avg=4096.0000 max=4194 max_rank=7 min=3940 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[567, 501, 476, 523, 503, 517, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 524, 532, 494, 563, 548, 498, 4, 1, 0, 0, 0, 0, 0, 0, 482, 480, 503, 532, 512, 514, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 493, 505, 531, 498, 522, 490, 24, 0, 0, 0, 0, 0, 0, 0, 492, 513, 481, 520, 503, 526, 533, 518, 18, 0, 0, 0, 0, 0, 0, 0, 476, 517, 508, 493, 558, 531, 533, 539, 9, 4, 4, 0, 0, 0, 0, 0, 513, 518, 522, 506, 526, 503, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 461, 526, 466, 514, 506, 456, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4195, 4049, 4077, 4104, 4172, 4127, 3943] avg=4096.0000 max=4195 max_rank=1 min=3943 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[527, 475, 481, 525, 519, 501, 515, 533, 516, 555, 490, 508, 511, 527, 506, 499, 543, 503, 522, 488, 513, 500, 514, 513, 502, 517, 470, 509, 490, 504, 506, 524, 527, 505, 474, 543, 495, 503, 504, 571, 509, 542, 508, 545, 518, 482, 522, 529, 492, 533, 510, 510, 501, 511, 545, 504, 531, 512, 522, 464, 517, 531, 510, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[520, 530, 494, 512, 471, 552, 523, 493, 537, 524, 499, 512, 504, 551, 491, 534, 489, 537, 463, 504, 500, 477, 507, 518, 548, 461, 525, 534, 495, 492, 577, 528, 487, 489, 520, 496, 508, 536, 512, 510, 484, 474, 537, 458, 497, 526, 514, 516, 486, 531, 519, 524, 512, 539, 517, 540, 534, 493, 511, 529, 534, 531, 506, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[462, 511, 539, 523, 461, 524, 516, 522, 518, 543, 536, 488, 500, 522, 508, 514, 486, 470, 545, 492, 489, 526, 525, 512, 488, 512, 464, 489, 490, 504, 502, 511, 512, 493, 498, 539, 497, 528, 534, 519, 493, 504, 545, 527, 509, 480, 549, 482, 544, 549, 527, 517, 525, 531, 532, 500, 516, 544, 528, 526, 521, 524, 478, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[514, 524, 526, 526, 508, 496, 505, 499, 500, 524, 527, 492, 494, 500, 517, 523, 498, 515, 508, 473, 507, 511, 534, 530, 542, 541, 545, 522, 476, 507, 531, 531, 479, 521, 514, 508, 493, 525, 494, 514, 520, 500, 522, 503, 526, 570, 518, 490, 509, 514, 487, 510, 467, 506, 482, 529, 515, 536, 488, 502, 540, 528, 510, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[522, 507, 500, 504, 525, 538, 504, 508, 492, 549, 483, 450, 494, 530, 514, 519, 518, 505, 544, 495, 509, 458, 500, 500, 517, 505, 507, 528, 551, 528, 514, 557, 545, 500, 507, 496, 512, 479, 490, 507, 514, 535, 505, 520, 536, 520, 514, 504, 511, 520, 452, 508, 535, 503, 518, 556, 522, 502, 500, 515, 500, 535, 507, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[509, 534, 482, 504, 515, 474, 514, 542, 506, 525, 543, 490, 503, 502, 491, 501, 536, 527, 491, 492, 507, 539, 517, 501, 496, 503, 500, 532, 494, 518, 492, 535, 521, 497, 524, 539, 517, 515, 470, 492, 550, 532, 549, 552, 490, 515, 486, 512, 516, 499, 550, 521, 489, 548, 492, 523, 501, 500, 511, 453, 527, 532, 519, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[474, 550, 546, 547, 501, 516, 521, 485, 499, 537, 502, 505, 508, 518, 522, 529, 499, 556, 488, 512, 539, 515, 488, 511, 495, 517, 479, 547, 507, 522, 521, 512, 504, 503, 499, 529, 495, 514, 485, 476, 539, 549, 492, 551, 524, 471, 523, 499, 493, 525, 506, 532, 487, 507, 537, 526, 503, 483, 502, 534, 484, 492, 502, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[517, 515, 502, 518, 511, 482, 483, 508, 553, 513, 536, 518, 543, 510, 502, 526, 549, 532, 524, 555, 493, 471, 529, 503, 512, 499, 508, 531, 511, 494, 482, 501, 480, 543, 527, 480, 544, 543, 496, 485, 483, 510, 538, 514, 495, 513, 551, 498, 510, 513, 531, 513, 529, 496, 541, 473, 506, 515, 477, 524, 539, 495, 490, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 475, 481, 525, 519, 501, 515, 533, 6, 4, 0, 0, 0, 0, 0, 0, 516, 541, 490, 508, 511, 527, 506, 499, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 522, 488, 513, 500, 514, 513, 18, 1, 0, 0, 0, 0, 0, 0, 502, 517, 470, 509, 490, 504, 506, 524, 2, 0, 0, 0, 0, 0, 0, 0, 527, 505, 474, 543, 495, 503, 504, 571, 24, 0, 0, 0, 0, 0, 0, 0, 509, 542, 483, 545, 518, 482, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 492, 509, 510, 510, 501, 511, 545, 504, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 522, 464, 517, 531, 510, 492, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4098, 4115, 4024, 4146, 4130, 4082, 4087] avg=4096.0000 max=4146 max_rank=4 min=4024 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 524, 526, 526, 508, 496, 505, 499, 6, 3, 0, 0, 0, 0, 0, 0, 500, 512, 527, 492, 494, 500, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 508, 473, 507, 511, 534, 530, 17, 1, 0, 0, 0, 0, 0, 0, 542, 541, 545, 522, 476, 507, 531, 531, 1, 0, 0, 0, 0, 0, 0, 0, 479, 521, 514, 508, 493, 525, 494, 514, 25, 0, 0, 0, 0, 0, 0, 0, 520, 500, 496, 503, 526, 570, 518, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 487, 510, 467, 506, 482, 529, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 488, 502, 540, 528, 510, 502, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4065, 4094, 4196, 4073, 4123, 3981, 4129] avg=4096.0000 max=4196 max_rank=3 min=3981 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[520, 530, 494, 512, 471, 552, 523, 493, 6, 3, 0, 0, 0, 0, 0, 0, 537, 512, 499, 512, 504, 551, 491, 534, 0, 0, 0, 0, 0, 0, 0, 0, 489, 537, 463, 504, 500, 477, 507, 518, 18, 1, 0, 0, 0, 0, 0, 0, 548, 461, 525, 534, 495, 492, 577, 528, 1, 0, 0, 0, 0, 0, 0, 0, 487, 489, 520, 496, 508, 536, 512, 510, 26, 0, 0, 0, 0, 0, 0, 0, 484, 474, 510, 458, 497, 526, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 519, 524, 512, 539, 517, 540, 0, 0, 0, 0, 0, 0, 0, 0, 534, 493, 511, 529, 534, 531, 506, 496, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4140, 4014, 4161, 4084, 3979, 4144, 4142] avg=4096.0000 max=4161 max_rank=3 min=3979 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[522, 507, 500, 504, 525, 538, 504, 508, 6, 4, 0, 0, 0, 0, 0, 0, 492, 535, 483, 450, 494, 530, 514, 519, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 544, 495, 509, 458, 500, 500, 17, 1, 0, 0, 0, 0, 0, 0, 517, 505, 507, 528, 551, 528, 514, 557, 2, 0, 0, 0, 0, 0, 0, 0, 545, 500, 507, 496, 512, 479, 490, 507, 24, 0, 0, 0, 0, 0, 0, 0, 514, 535, 480, 520, 536, 520, 514, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 452, 508, 535, 503, 518, 556, 0, 0, 0, 0, 0, 0, 0, 0, 522, 502, 500, 515, 500, 535, 507, 525, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4017, 4047, 4209, 4060, 4123, 4080, 4114] avg=4096.0000 max=4209 max_rank=3 min=4017 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[474, 550, 546, 547, 501, 516, 521, 485, 6, 4, 0, 0, 0, 0, 0, 0, 499, 523, 502, 505, 508, 518, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 499, 556, 488, 512, 539, 515, 488, 511, 18, 1, 0, 0, 0, 0, 0, 0, 495, 517, 479, 547, 507, 522, 521, 512, 2, 0, 0, 0, 0, 0, 0, 0, 504, 503, 499, 529, 495, 514, 485, 476, 24, 0, 0, 0, 0, 0, 0, 0, 539, 549, 467, 551, 524, 471, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 506, 532, 487, 507, 537, 526, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 502, 534, 484, 492, 502, 534, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4106, 4127, 4102, 4029, 4123, 4089, 4042] avg=4096.0000 max=4150 max_rank=0 min=4029 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 511, 539, 523, 461, 524, 516, 522, 6, 4, 0, 0, 0, 0, 0, 0, 518, 529, 536, 488, 500, 522, 508, 514, 0, 0, 0, 0, 0, 0, 0, 0, 486, 470, 545, 492, 489, 526, 525, 512, 18, 1, 0, 0, 0, 0, 0, 0, 488, 512, 464, 489, 490, 504, 502, 511, 2, 0, 0, 0, 0, 0, 0, 0, 512, 493, 498, 539, 497, 528, 534, 519, 26, 0, 0, 0, 0, 0, 0, 0, 493, 504, 518, 527, 509, 480, 549, 482, 0, 0, 0, 0, 0, 0, 0, 0, 544, 525, 527, 517, 525, 531, 532, 500, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 528, 526, 521, 524, 478, 505, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4115, 4064, 3962, 4146, 4062, 4201, 4150] avg=4096.0000 max=4201 max_rank=6 min=3962 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 534, 482, 504, 515, 474, 514, 542, 6, 3, 0, 0, 0, 0, 0, 0, 506, 513, 543, 490, 503, 502, 491, 501, 0, 0, 0, 0, 0, 0, 0, 0, 536, 527, 491, 492, 507, 539, 517, 501, 17, 1, 0, 0, 0, 0, 0, 0, 496, 503, 500, 532, 494, 518, 492, 535, 1, 0, 0, 0, 0, 0, 0, 0, 521, 497, 524, 539, 517, 515, 470, 492, 26, 0, 0, 0, 0, 0, 0, 0, 550, 532, 522, 552, 490, 515, 486, 512, 0, 0, 0, 0, 0, 0, 0, 0, 516, 476, 550, 521, 489, 548, 492, 523, 0, 0, 0, 0, 0, 0, 0, 0, 501, 500, 511, 453, 527, 532, 519, 511, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4049, 4128, 4071, 4101, 4159, 4115, 4062] avg=4096.0000 max=4159 max_rank=5 min=4049 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 515, 502, 518, 511, 482, 483, 508, 9, 7, 0, 0, 0, 0, 0, 0, 553, 490, 536, 518, 543, 510, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 549, 532, 524, 555, 493, 471, 529, 503, 21, 2, 0, 0, 0, 0, 0, 0, 512, 499, 508, 531, 511, 494, 482, 501, 5, 0, 0, 0, 0, 0, 0, 0, 480, 543, 527, 480, 544, 543, 496, 485, 31, 0, 0, 0, 0, 0, 0, 0, 483, 510, 505, 514, 495, 513, 551, 498, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 531, 513, 529, 496, 541, 473, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 477, 524, 539, 495, 490, 485, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4178, 4179, 4043, 4129, 4069, 4076, 4042] avg=4096.0000 max=4179 max_rank=2 min=4042 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[520, 476, 536, 497, 525, 492, 482, 529, 513, 504, 526, 492, 498, 490, 503, 496, 525, 509, 549, 513, 528, 508, 564, 495, 505, 511, 526, 518, 530, 496, 512, 496, 500, 510, 483, 501, 485, 525, 513, 481, 517, 511, 533, 506, 489, 523, 546, 547, 531, 504, 546, 494, 510, 489, 478, 546, 528, 524, 520, 522, 532, 515, 500, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[518, 500, 548, 542, 497, 466, 514, 507, 539, 521, 535, 471, 507, 509, 505, 541, 517, 523, 493, 523, 507, 503, 531, 501, 492, 529, 531, 488, 465, 521, 483, 517, 487, 503, 547, 527, 494, 523, 511, 536, 508, 498, 521, 538, 519, 479, 501, 484, 560, 514, 509, 472, 507, 483, 537, 520, 522, 486, 498, 477, 560, 526, 525, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[501, 527, 505, 533, 495, 482, 524, 504, 507, 509, 504, 489, 522, 482, 505, 553, 500, 508, 521, 550, 496, 529, 549, 506, 494, 543, 451, 509, 539, 512, 513, 494, 519, 520, 537, 532, 533, 509, 568, 500, 489, 518, 478, 508, 515, 486, 505, 510, 508, 548, 519, 529, 505, 480, 505, 497, 530, 500, 503, 510, 513, 519, 518, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[501, 457, 536, 513, 512, 493, 533, 495, 492, 477, 549, 524, 511, 519, 510, 517, 518, 524, 505, 487, 533, 512, 512, 481, 505, 521, 488, 540, 543, 504, 516, 515, 491, 526, 504, 488, 504, 504, 493, 547, 504, 513, 531, 507, 510, 495, 510, 516, 520, 512, 507, 499, 539, 557, 495, 516, 533, 530, 537, 491, 524, 514, 486, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[513, 552, 521, 469, 515, 508, 515, 472, 496, 507, 503, 496, 529, 526, 485, 523, 503, 472, 514, 509, 542, 495, 489, 551, 503, 539, 472, 564, 580, 519, 513, 543, 492, 486, 518, 531, 501, 518, 496, 526, 494, 538, 499, 487, 469, 558, 558, 481, 504, 520, 498, 505, 534, 534, 522, 499, 500, 530, 510, 498, 501, 523, 493, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[513, 485, 525, 515, 519, 508, 484, 519, 514, 492, 520, 470, 507, 488, 533, 497, 504, 502, 545, 524, 480, 518, 523, 522, 582, 514, 526, 511, 501, 495, 536, 548, 514, 491, 551, 522, 517, 490, 516, 495, 510, 480, 518, 506, 528, 535, 520, 501, 472, 509, 484, 520, 516, 491, 511, 516, 516, 523, 513, 522, 514, 512, 488, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[541, 485, 544, 559, 507, 524, 552, 520, 529, 490, 487, 488, 527, 489, 497, 491, 467, 518, 500, 486, 545, 515, 496, 506, 492, 514, 539, 496, 519, 490, 520, 510, 524, 539, 497, 509, 514, 507, 519, 471, 488, 538, 559, 497, 511, 547, 522, 494, 523, 505, 520, 538, 504, 532, 518, 514, 503, 495, 511, 504, 491, 514, 512, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[537, 498, 498, 517, 554, 476, 495, 504, 528, 505, 482, 557, 518, 516, 500, 525, 497, 559, 526, 509, 516, 515, 472, 510, 522, 514, 491, 509, 558, 511, 534, 513, 500, 516, 504, 479, 509, 504, 480, 518, 486, 545, 501, 547, 477, 502, 501, 498, 503, 511, 544, 532, 514, 515, 513, 509, 491, 520, 535, 521, 467, 492, 527, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 476, 536, 497, 525, 492, 482, 529, 6, 1, 0, 0, 0, 0, 0, 0, 513, 504, 526, 492, 498, 490, 503, 496, 27, 1, 0, 0, 0, 0, 0, 0, 525, 509, 539, 513, 528, 508, 564, 495, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 526, 518, 503, 496, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 483, 501, 485, 525, 513, 481, 10, 4, 0, 0, 0, 0, 0, 0, 517, 511, 533, 506, 489, 523, 546, 547, 6, 0, 0, 0, 0, 0, 0, 0, 531, 504, 546, 494, 499, 489, 478, 546, 0, 0, 0, 0, 0, 0, 0, 0, 528, 524, 520, 522, 532, 515, 500, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4050, 4181, 4067, 4012, 4178, 4087, 4129] avg=4096.0000 max=4181 max_rank=2 min=4012 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 500, 548, 542, 497, 466, 514, 507, 6, 1, 0, 0, 0, 0, 0, 0, 539, 521, 535, 471, 507, 509, 505, 541, 23, 1, 0, 0, 0, 0, 0, 0, 517, 523, 484, 523, 507, 503, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 492, 529, 531, 488, 442, 521, 483, 517, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 547, 527, 494, 523, 511, 536, 10, 3, 0, 0, 0, 0, 0, 0, 508, 498, 521, 538, 519, 479, 501, 484, 7, 0, 0, 0, 0, 0, 0, 0, 560, 514, 509, 472, 496, 483, 537, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 486, 498, 477, 560, 526, 525, 544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4152, 4089, 4003, 4141, 4055, 4091, 4138] avg=4096.0000 max=4152 max_rank=1 min=4003 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 457, 536, 513, 512, 493, 533, 495, 9, 6, 0, 0, 0, 0, 0, 0, 492, 477, 549, 524, 511, 519, 510, 517, 32, 8, 0, 0, 0, 0, 0, 0, 518, 524, 488, 487, 533, 512, 512, 481, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 488, 540, 511, 504, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 491, 526, 504, 488, 504, 504, 493, 547, 12, 8, 0, 0, 0, 0, 0, 0, 504, 513, 531, 507, 510, 495, 510, 516, 11, 0, 0, 0, 0, 0, 0, 0, 520, 512, 507, 499, 519, 557, 495, 516, 0, 0, 0, 0, 0, 0, 0, 0, 533, 530, 537, 491, 524, 514, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4139, 4055, 4100, 4077, 4097, 4125, 4120] avg=4096.0000 max=4139 max_rank=1 min=4055 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 485, 525, 515, 519, 508, 484, 519, 6, 1, 0, 0, 0, 0, 0, 0, 514, 492, 520, 470, 507, 488, 533, 497, 25, 1, 0, 0, 0, 0, 0, 0, 504, 502, 535, 524, 480, 518, 523, 522, 0, 0, 0, 0, 0, 0, 0, 0, 582, 514, 526, 511, 476, 495, 536, 548, 0, 0, 0, 0, 0, 0, 0, 0, 514, 491, 551, 522, 517, 490, 516, 495, 10, 4, 0, 0, 0, 0, 0, 0, 510, 480, 518, 506, 528, 535, 520, 501, 7, 0, 0, 0, 0, 0, 0, 0, 472, 509, 484, 520, 505, 491, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 513, 522, 514, 512, 488, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4047, 4108, 4188, 4110, 4105, 4008, 4127] avg=4096.0000 max=4188 max_rank=3 min=4008 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 498, 498, 517, 554, 476, 495, 504, 6, 1, 0, 0, 0, 0, 0, 0, 528, 505, 482, 557, 518, 516, 500, 525, 28, 1, 0, 0, 0, 0, 0, 0, 497, 559, 517, 509, 516, 515, 472, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 491, 509, 530, 511, 534, 513, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 504, 479, 509, 504, 480, 518, 10, 3, 0, 0, 0, 0, 0, 0, 486, 545, 501, 547, 477, 502, 501, 498, 7, 0, 0, 0, 0, 0, 0, 0, 503, 511, 544, 532, 503, 515, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 491, 520, 535, 521, 467, 492, 527, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4160, 4095, 4124, 4023, 4064, 4130, 4086] avg=4096.0000 max=4160 max_rank=1 min=4023 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 527, 505, 533, 495, 482, 524, 504, 6, 1, 0, 0, 0, 0, 0, 0, 507, 509, 504, 489, 522, 482, 505, 553, 27, 1, 0, 0, 0, 0, 0, 0, 500, 508, 512, 550, 496, 529, 549, 506, 0, 0, 0, 0, 0, 0, 0, 0, 494, 543, 451, 509, 512, 512, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 537, 532, 533, 509, 568, 500, 10, 3, 0, 0, 0, 0, 0, 0, 489, 518, 478, 508, 515, 486, 505, 510, 6, 0, 0, 0, 0, 0, 0, 0, 508, 548, 519, 529, 494, 480, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 503, 510, 513, 519, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4099, 4150, 4028, 4231, 4015, 4080, 4087] avg=4096.0000 max=4231 max_rank=4 min=4015 min_rank=5 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 552, 521, 469, 515, 508, 515, 472, 6, 1, 0, 0, 0, 0, 0, 0, 496, 507, 503, 496, 529, 526, 485, 523, 29, 2, 0, 0, 0, 0, 0, 0, 503, 472, 505, 509, 542, 495, 489, 551, 0, 0, 0, 0, 0, 0, 0, 0, 503, 539, 472, 564, 551, 519, 513, 543, 0, 0, 0, 0, 0, 0, 0, 0, 492, 486, 518, 531, 501, 518, 496, 526, 10, 3, 0, 0, 0, 0, 0, 0, 494, 538, 499, 487, 469, 558, 558, 481, 7, 0, 0, 0, 0, 0, 0, 0, 504, 520, 498, 505, 522, 534, 522, 499, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 510, 498, 501, 523, 493, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4096, 4066, 4204, 4081, 4091, 4104, 4054] avg=4096.0000 max=4204 max_rank=3 min=4054 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[541, 485, 544, 559, 507, 524, 552, 520, 6, 1, 0, 0, 0, 0, 0, 0, 529, 490, 487, 488, 527, 489, 497, 491, 26, 1, 0, 0, 0, 0, 0, 0, 467, 518, 491, 486, 545, 515, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 539, 496, 493, 490, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 524, 539, 497, 509, 514, 507, 519, 471, 10, 3, 0, 0, 0, 0, 0, 0, 488, 538, 559, 497, 511, 547, 522, 494, 7, 0, 0, 0, 0, 0, 0, 0, 523, 505, 520, 538, 493, 532, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 511, 504, 491, 514, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4239, 4025, 4024, 4054, 4093, 4163, 4143, 4027] avg=4096.0000 max=4239 max_rank=0 min=4024 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[496, 527, 517, 461, 518, 506, 517, 507, 511, 497, 550, 505, 525, 515, 495, 516, 469, 518, 538, 494, 532, 490, 518, 504, 481, 490, 545, 531, 529, 502, 551, 510, 519, 504, 524, 514, 500, 501, 502, 549, 499, 539, 534, 558, 539, 469, 491, 532, 479, 540, 526, 494, 541, 500, 491, 512, 524, 519, 491, 516, 495, 491, 513, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[510, 520, 492, 521, 508, 469, 494, 536, 521, 511, 512, 516, 482, 545, 529, 540, 486, 534, 517, 521, 484, 480, 553, 511, 536, 482, 487, 534, 500, 514, 517, 510, 495, 493, 539, 517, 500, 540, 525, 494, 534, 542, 523, 544, 534, 513, 508, 482, 514, 551, 508, 501, 505, 497, 520, 511, 473, 533, 510, 523, 500, 492, 487, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[508, 491, 554, 521, 528, 471, 505, 511, 504, 520, 492, 535, 526, 473, 470, 515, 488, 511, 527, 511, 565, 513, 535, 482, 550, 492, 511, 533, 492, 501, 475, 515, 492, 489, 564, 513, 525, 516, 534, 500, 570, 534, 503, 500, 528, 475, 532, 484, 547, 487, 533, 490, 488, 495, 535, 511, 483, 536, 538, 539, 493, 499, 498, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[504, 531, 493, 500, 548, 512, 489, 540, 491, 476, 505, 517, 534, 525, 532, 492, 519, 504, 535, 517, 485, 544, 498, 526, 527, 547, 491, 503, 535, 544, 449, 505, 510, 488, 503, 503, 502, 510, 511, 534, 511, 512, 551, 530, 502, 518, 477, 524, 517, 452, 491, 508, 496, 518, 535, 504, 490, 524, 528, 504, 544, 525, 558, 470] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[493, 506, 518, 498, 488, 510, 528, 508, 548, 487, 519, 525, 510, 535, 510, 499, 529, 540, 455, 513, 507, 539, 517, 534, 488, 503, 555, 500, 495, 508, 528, 512, 507, 487, 494, 489, 501, 521, 481, 521, 507, 561, 530, 509, 499, 551, 514, 530, 511, 526, 489, 534, 513, 517, 488, 483, 486, 515, 544, 483, 520, 544, 512, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[500, 510, 514, 485, 513, 498, 516, 510, 498, 561, 536, 464, 509, 514, 535, 503, 526, 501, 488, 493, 547, 499, 517, 562, 475, 501, 543, 481, 534, 539, 499, 531, 527, 476, 486, 535, 504, 534, 491, 526, 488, 513, 496, 506, 500, 544, 534, 492, 517, 505, 528, 547, 537, 490, 532, 533, 501, 518, 540, 502, 463, 513, 496, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[505, 485, 525, 549, 496, 505, 513, 497, 499, 486, 479, 491, 508, 515, 500, 509, 523, 509, 533, 502, 537, 491, 527, 496, 556, 515, 494, 555, 500, 504, 546, 517, 518, 489, 544, 517, 493, 513, 492, 524, 475, 516, 493, 512, 511, 491, 515, 510, 519, 498, 521, 532, 519, 485, 507, 516, 546, 487, 534, 521, 523, 557, 464, 559] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[492, 502, 501, 531, 514, 526, 528, 505, 520, 482, 480, 493, 526, 536, 475, 520, 498, 507, 504, 535, 543, 482, 510, 513, 499, 548, 523, 506, 475, 487, 503, 532, 498, 541, 516, 505, 512, 552, 538, 500, 486, 482, 505, 491, 545, 535, 511, 519, 538, 552, 491, 538, 468, 489, 480, 497, 553, 512, 504, 520, 552, 499, 539, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 491, 554, 521, 528, 471, 505, 511, 24, 0, 0, 0, 0, 0, 0, 0, 504, 520, 492, 535, 526, 473, 470, 515, 15, 0, 0, 0, 0, 0, 0, 0, 488, 511, 527, 511, 550, 513, 535, 482, 0, 0, 0, 0, 0, 0, 0, 0, 550, 492, 503, 533, 492, 501, 475, 515, 0, 0, 0, 0, 0, 0, 0, 0, 492, 489, 564, 513, 525, 516, 534, 500, 3, 0, 0, 0, 0, 0, 0, 0, 570, 510, 503, 500, 528, 475, 532, 484, 0, 0, 0, 0, 0, 0, 0, 0, 547, 487, 533, 490, 488, 495, 535, 511, 8, 0, 0, 0, 0, 0, 0, 0, 483, 536, 535, 539, 493, 499, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4050, 4117, 4061, 4136, 4102, 4094, 4095] avg=4096.0000 max=4136 max_rank=4 min=4050 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16427 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 520, 492, 521, 508, 469, 494, 536, 25, 0, 0, 0, 0, 0, 0, 0, 521, 511, 512, 516, 482, 545, 529, 540, 13, 0, 0, 0, 0, 0, 0, 0, 486, 534, 517, 521, 471, 480, 553, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 482, 479, 534, 500, 514, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 495, 493, 539, 517, 500, 540, 525, 494, 2, 0, 0, 0, 0, 0, 0, 0, 534, 517, 523, 544, 534, 513, 508, 482, 0, 0, 0, 0, 0, 0, 0, 0, 514, 551, 508, 501, 505, 497, 520, 511, 8, 0, 0, 0, 0, 0, 0, 0, 473, 533, 508, 523, 500, 492, 487, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4169, 4073, 4072, 4105, 4155, 4115, 4004] avg=4096.0000 max=4169 max_rank=1 min=4004 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16379 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 527, 517, 461, 518, 506, 517, 507, 24, 0, 0, 0, 0, 0, 0, 0, 511, 497, 550, 505, 525, 515, 495, 516, 14, 0, 0, 0, 0, 0, 0, 0, 469, 518, 538, 494, 518, 490, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 481, 490, 536, 531, 529, 502, 551, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 524, 514, 500, 501, 502, 549, 2, 0, 0, 0, 0, 0, 0, 0, 499, 515, 534, 558, 539, 469, 491, 532, 0, 0, 0, 0, 0, 0, 0, 0, 479, 540, 526, 494, 541, 500, 491, 512, 9, 0, 0, 0, 0, 0, 0, 0, 524, 519, 489, 516, 495, 491, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4128, 4049, 4130, 4115, 4137, 4092, 4044] avg=4096.0000 max=4137 max_rank=5 min=4044 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[504, 531, 493, 500, 548, 512, 489, 540, 23, 0, 0, 0, 0, 0, 0, 0, 491, 476, 505, 517, 534, 525, 532, 492, 13, 0, 0, 0, 0, 0, 0, 0, 519, 504, 535, 517, 472, 544, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 527, 547, 483, 503, 535, 544, 449, 505, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 503, 503, 502, 510, 511, 534, 3, 0, 0, 0, 0, 0, 0, 0, 511, 489, 551, 530, 502, 518, 477, 524, 0, 0, 0, 0, 0, 0, 0, 0, 517, 452, 491, 508, 496, 518, 535, 504, 8, 0, 0, 0, 0, 0, 0, 0, 490, 524, 525, 504, 544, 525, 558, 470, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4085, 4115, 4093, 4064, 4102, 4029, 4140] avg=4096.0000 max=4140 max_rank=0 min=4029 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16335 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[500, 510, 514, 485, 513, 498, 516, 510, 23, 0, 0, 0, 0, 0, 0, 0, 498, 561, 536, 464, 509, 514, 535, 503, 15, 0, 0, 0, 0, 0, 0, 0, 526, 501, 488, 493, 532, 499, 517, 562, 0, 0, 0, 0, 0, 0, 0, 0, 475, 501, 534, 481, 534, 539, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 527, 476, 486, 535, 504, 534, 491, 526, 3, 0, 0, 0, 0, 0, 0, 0, 488, 490, 496, 506, 500, 544, 534, 492, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 528, 547, 537, 490, 532, 533, 9, 0, 0, 0, 0, 0, 0, 0, 501, 518, 537, 502, 463, 513, 496, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4135, 4118, 4094, 4082, 4050, 4198, 4022] avg=4096.0000 max=4198 max_rank=6 min=4022 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16352 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 506, 518, 498, 488, 510, 528, 508, 30, 0, 0, 0, 0, 0, 0, 0, 548, 487, 519, 525, 510, 535, 510, 499, 18, 1, 0, 0, 0, 0, 0, 0, 529, 540, 455, 513, 489, 539, 517, 534, 0, 0, 0, 0, 0, 0, 0, 0, 488, 503, 540, 500, 495, 508, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 494, 489, 501, 521, 481, 521, 6, 2, 0, 0, 0, 0, 0, 0, 507, 530, 530, 509, 499, 551, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 489, 534, 513, 517, 488, 483, 13, 0, 0, 0, 0, 0, 0, 0, 486, 515, 538, 483, 520, 544, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4152, 4116, 4074, 4009, 4170, 4074, 4094] avg=4096.0000 max=4170 max_rank=5 min=4009 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16347 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 485, 525, 549, 496, 505, 513, 497, 23, 0, 0, 0, 0, 0, 0, 0, 499, 486, 479, 491, 508, 515, 500, 509, 15, 0, 0, 0, 0, 0, 0, 0, 523, 509, 533, 502, 522, 491, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 556, 515, 486, 555, 500, 504, 546, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 544, 517, 493, 513, 492, 524, 3, 0, 0, 0, 0, 0, 0, 0, 475, 493, 493, 512, 511, 491, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 521, 532, 519, 485, 507, 516, 8, 0, 0, 0, 0, 0, 0, 0, 546, 487, 531, 521, 523, 557, 464, 559, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4002, 4103, 4179, 4093, 4000, 4105, 4188] avg=4096.0000 max=4188 max_rank=7 min=4000 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[492, 502, 501, 531, 514, 526, 528, 505, 22, 0, 0, 0, 0, 0, 0, 0, 520, 482, 480, 493, 526, 536, 475, 520, 15, 0, 0, 0, 0, 0, 0, 0, 498, 507, 504, 535, 528, 482, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 499, 548, 515, 506, 475, 487, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 498, 541, 516, 505, 512, 552, 538, 500, 2, 0, 0, 0, 0, 0, 0, 0, 486, 460, 505, 491, 545, 535, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 538, 552, 491, 538, 468, 489, 480, 497, 8, 0, 0, 0, 0, 0, 0, 0, 553, 512, 502, 520, 552, 499, 539, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4047, 4077, 4065, 4164, 4052, 4061, 4181] avg=4096.0000 max=4181 max_rank=7 min=4047 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16458 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[502, 487, 513, 567, 535, 516, 515, 541, 509, 496, 543, 519, 522, 505, 491, 515, 492, 502, 488, 497, 538, 562, 522, 519, 523, 522, 531, 543, 548, 491, 467, 491, 482, 502, 536, 486, 506, 496, 538, 486, 499, 530, 514, 517, 456, 488, 507, 530, 491, 526, 542, 527, 501, 521, 469, 527, 550, 498, 503, 485, 470, 548, 554, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[550, 521, 516, 546, 495, 527, 497, 489, 511, 492, 515, 480, 487, 514, 538, 481, 540, 526, 552, 505, 534, 542, 504, 516, 539, 535, 518, 514, 530, 524, 507, 541, 480, 461, 483, 481, 502, 480, 516, 533, 504, 509, 547, 502, 524, 510, 499, 493, 467, 470, 490, 512, 545, 505, 531, 507, 521, 475, 477, 515, 539, 541, 539, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[497, 498, 506, 507, 492, 508, 522, 496, 481, 462, 499, 511, 505, 519, 499, 572, 560, 517, 521, 515, 531, 502, 495, 534, 524, 514, 447, 570, 526, 486, 495, 492, 534, 504, 523, 488, 508, 512, 527, 505, 494, 541, 521, 502, 504, 519, 469, 512, 509, 499, 516, 529, 531, 527, 476, 478, 540, 521, 542, 519, 570, 497, 521, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[508, 485, 499, 522, 543, 521, 522, 537, 523, 516, 537, 495, 559, 530, 502, 483, 531, 528, 506, 501, 516, 512, 508, 492, 511, 491, 510, 518, 486, 496, 545, 494, 494, 510, 517, 536, 521, 473, 544, 511, 487, 497, 530, 493, 505, 518, 518, 514, 526, 513, 481, 532, 532, 492, 507, 518, 515, 516, 497, 495, 511, 526, 536, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[495, 510, 538, 482, 543, 506, 547, 519, 501, 500, 495, 489, 473, 512, 546, 506, 510, 494, 509, 503, 531, 498, 487, 513, 548, 474, 499, 518, 464, 523, 459, 543, 515, 503, 503, 539, 534, 527, 518, 499, 516, 520, 525, 504, 518, 542, 555, 464, 516, 510, 523, 516, 521, 527, 513, 515, 503, 553, 527, 518, 499, 509, 493, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[508, 510, 531, 514, 534, 547, 531, 523, 503, 486, 514, 486, 512, 498, 499, 499, 535, 533, 517, 507, 528, 510, 509, 518, 496, 488, 481, 504, 526, 530, 533, 502, 504, 512, 505, 511, 465, 548, 492, 524, 525, 536, 514, 520, 482, 522, 540, 524, 520, 495, 490, 513, 520, 575, 543, 506, 517, 475, 497, 486, 468, 516, 474, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[510, 536, 485, 542, 536, 508, 499, 525, 483, 535, 540, 545, 519, 531, 516, 520, 494, 528, 478, 574, 489, 543, 520, 504, 499, 506, 492, 486, 521, 552, 506, 512, 524, 533, 484, 491, 516, 499, 491, 541, 517, 487, 529, 470, 496, 492, 489, 521, 527, 504, 525, 505, 514, 500, 526, 508, 501, 471, 531, 536, 467, 514, 496, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[510, 513, 506, 492, 511, 528, 497, 518, 490, 495, 561, 534, 502, 526, 528, 489, 509, 513, 510, 475, 577, 517, 478, 526, 516, 509, 489, 486, 518, 476, 509, 548, 528, 487, 522, 521, 529, 511, 516, 502, 536, 501, 499, 486, 536, 536, 515, 478, 544, 506, 528, 495, 509, 493, 484, 515, 533, 515, 503, 539, 516, 541, 509, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 487, 513, 567, 502, 516, 515, 541, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 543, 519, 522, 505, 491, 515, 15, 0, 0, 0, 0, 0, 0, 0, 492, 502, 488, 497, 504, 562, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 531, 543, 548, 491, 467, 491, 9, 6, 0, 0, 0, 0, 0, 0, 482, 502, 536, 486, 506, 496, 538, 486, 25, 0, 0, 0, 0, 0, 0, 0, 499, 530, 514, 517, 456, 488, 507, 530, 12, 1, 1, 0, 0, 0, 0, 0, 491, 526, 542, 527, 500, 521, 469, 527, 0, 0, 0, 0, 0, 0, 0, 0, 550, 498, 503, 485, 470, 547, 554, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4115, 4086, 4131, 4057, 4055, 4103, 4078] avg=4096.0000 max=4143 max_rank=0 min=4055 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[497, 498, 506, 507, 462, 508, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 481, 462, 499, 511, 505, 519, 499, 572, 14, 0, 0, 0, 0, 0, 0, 0, 560, 517, 521, 515, 498, 502, 495, 534, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 447, 570, 526, 486, 495, 492, 9, 5, 0, 0, 0, 0, 0, 0, 534, 504, 523, 488, 508, 512, 527, 505, 24, 0, 0, 0, 0, 0, 0, 0, 494, 541, 521, 502, 504, 519, 469, 512, 11, 1, 1, 0, 0, 0, 0, 0, 509, 499, 516, 529, 530, 527, 476, 478, 0, 0, 0, 0, 0, 0, 0, 0, 540, 521, 542, 519, 570, 496, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4062, 4142, 4068, 4125, 4075, 4064, 4236] avg=4096.0000 max=4236 max_rank=7 min=3996 min_rank=0 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 485, 499, 522, 509, 521, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 537, 495, 559, 530, 502, 483, 16, 0, 0, 0, 0, 0, 0, 0, 531, 528, 506, 501, 483, 512, 508, 492, 0, 0, 0, 0, 0, 0, 0, 0, 511, 491, 510, 518, 486, 496, 545, 494, 9, 6, 0, 0, 0, 0, 0, 0, 494, 510, 517, 536, 521, 473, 544, 511, 24, 0, 0, 0, 0, 0, 0, 0, 487, 497, 530, 493, 505, 518, 518, 514, 12, 1, 1, 0, 0, 0, 0, 0, 526, 513, 481, 532, 531, 492, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 497, 495, 511, 525, 536, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4161, 4061, 4066, 4130, 4076, 4100, 4071] avg=4096.0000 max=4161 max_rank=1 min=4061 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[550, 521, 516, 546, 465, 527, 497, 489, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 515, 480, 487, 514, 538, 481, 14, 0, 0, 0, 0, 0, 0, 0, 540, 526, 552, 505, 500, 542, 504, 516, 0, 0, 0, 0, 0, 0, 0, 0, 539, 535, 518, 514, 530, 524, 507, 541, 9, 5, 0, 0, 0, 0, 0, 0, 480, 461, 483, 481, 502, 480, 516, 533, 25, 0, 0, 0, 0, 0, 0, 0, 504, 509, 547, 502, 524, 510, 499, 493, 11, 1, 1, 0, 0, 0, 0, 0, 467, 470, 490, 512, 544, 505, 531, 507, 0, 0, 0, 0, 0, 0, 0, 0, 521, 475, 477, 515, 539, 540, 539, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4032, 4185, 4222, 3961, 4101, 4026, 4130] avg=4096.0000 max=4222 max_rank=3 min=3961 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[495, 510, 538, 482, 509, 506, 547, 519, 0, 0, 0, 0, 0, 0, 0, 0, 501, 500, 495, 489, 473, 512, 546, 506, 16, 0, 0, 0, 0, 0, 0, 0, 510, 494, 509, 503, 498, 498, 487, 513, 0, 0, 0, 0, 0, 0, 0, 0, 548, 474, 499, 518, 464, 523, 459, 543, 9, 6, 0, 0, 0, 0, 0, 0, 515, 503, 503, 539, 534, 527, 518, 499, 24, 0, 0, 0, 0, 0, 0, 0, 516, 520, 525, 504, 518, 542, 555, 464, 12, 1, 1, 0, 0, 0, 0, 0, 516, 510, 523, 516, 520, 527, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 553, 527, 518, 499, 508, 493, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4038, 4012, 4043, 4162, 4158, 4140, 4109] avg=4096.0000 max=4162 max_rank=4 min=4012 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 510, 531, 514, 502, 547, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 503, 486, 514, 486, 512, 498, 499, 499, 15, 0, 0, 0, 0, 0, 0, 0, 535, 533, 517, 507, 495, 510, 509, 518, 0, 0, 0, 0, 0, 0, 0, 0, 496, 488, 481, 504, 526, 530, 533, 502, 9, 6, 0, 0, 0, 0, 0, 0, 504, 512, 505, 511, 465, 548, 492, 524, 24, 0, 0, 0, 0, 0, 0, 0, 525, 536, 514, 520, 482, 522, 540, 524, 11, 1, 1, 0, 0, 0, 0, 0, 520, 495, 490, 513, 519, 575, 543, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 497, 486, 468, 515, 474, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4012, 4124, 4075, 4085, 4176, 4161, 3969] avg=4096.0000 max=4176 max_rank=5 min=3969 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 536, 485, 542, 494, 508, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 483, 535, 540, 545, 519, 531, 516, 520, 19, 0, 0, 0, 0, 0, 0, 0, 494, 528, 478, 574, 449, 543, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 492, 486, 521, 552, 506, 512, 14, 9, 0, 0, 0, 0, 0, 0, 524, 533, 484, 491, 516, 499, 491, 541, 26, 0, 0, 0, 0, 0, 0, 0, 517, 487, 529, 470, 496, 492, 489, 521, 14, 8, 4, 0, 0, 0, 0, 0, 527, 504, 525, 505, 506, 500, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 501, 471, 531, 536, 467, 510, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4208, 4090, 4097, 4105, 4027, 4101, 4041] avg=4096.0000 max=4208 max_rank=1 min=4027 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 513, 506, 492, 480, 528, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 490, 495, 561, 534, 502, 526, 528, 489, 15, 0, 0, 0, 0, 0, 0, 0, 509, 513, 510, 475, 540, 517, 478, 526, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 489, 486, 518, 476, 509, 548, 10, 5, 0, 0, 0, 0, 0, 0, 528, 487, 522, 521, 529, 511, 516, 502, 27, 0, 0, 0, 0, 0, 0, 0, 536, 501, 499, 486, 536, 536, 515, 478, 11, 1, 1, 0, 0, 0, 0, 0, 544, 506, 528, 495, 508, 493, 484, 515, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 503, 539, 516, 540, 509, 479, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4140, 4068, 4066, 4143, 4100, 4073, 4134] avg=4096.0000 max=4143 max_rank=4 min=4044 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[520, 516, 512, 485, 474, 480, 521, 510, 528, 519, 507, 510, 535, 521, 505, 524, 547, 528, 526, 499, 519, 493, 518, 515, 517, 509, 507, 544, 531, 553, 499, 518, 475, 517, 518, 502, 541, 496, 504, 507, 507, 508, 458, 483, 496, 504, 505, 507, 502, 529, 532, 545, 515, 474, 480, 568, 482, 494, 523, 502, 554, 532, 510, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[503, 469, 519, 513, 517, 479, 561, 512, 523, 520, 500, 502, 489, 485, 535, 554, 471, 542, 485, 530, 478, 498, 527, 526, 492, 519, 522, 509, 509, 521, 502, 548, 517, 496, 517, 519, 501, 527, 498, 531, 544, 505, 512, 518, 522, 490, 464, 525, 455, 526, 509, 535, 526, 528, 480, 543, 546, 501, 503, 517, 546, 518, 498, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[485, 513, 532, 500, 531, 531, 572, 525, 486, 543, 523, 531, 534, 535, 489, 532, 491, 551, 515, 539, 538, 540, 470, 496, 519, 507, 510, 570, 532, 509, 509, 557, 529, 507, 513, 492, 537, 522, 456, 486, 452, 485, 481, 491, 487, 465, 525, 507, 471, 506, 487, 521, 531, 526, 534, 532, 514, 520, 497, 496, 502, 509, 489, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[540, 495, 526, 547, 489, 499, 491, 511, 498, 489, 507, 514, 461, 485, 489, 540, 495, 497, 509, 516, 534, 524, 501, 536, 518, 544, 478, 524, 487, 501, 535, 495, 529, 502, 508, 497, 533, 509, 511, 525, 502, 519, 535, 546, 493, 547, 540, 512, 514, 517, 480, 500, 503, 489, 526, 506, 526, 527, 545, 493, 561, 484, 517, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[479, 506, 507, 535, 486, 500, 505, 509, 556, 524, 504, 504, 466, 554, 544, 523, 489, 530, 509, 493, 539, 492, 514, 476, 492, 526, 524, 502, 515, 495, 534, 470, 523, 500, 500, 503, 511, 504, 505, 526, 497, 556, 532, 505, 518, 492, 553, 502, 486, 488, 518, 551, 497, 517, 518, 511, 530, 508, 543, 481, 536, 511, 541, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[514, 522, 561, 498, 515, 487, 528, 527, 478, 536, 499, 517, 478, 527, 530, 534, 470, 524, 492, 536, 536, 530, 508, 507, 515, 511, 501, 510, 552, 496, 494, 464, 524, 514, 508, 526, 520, 491, 491, 490, 502, 551, 541, 539, 520, 515, 526, 510, 509, 525, 521, 502, 529, 534, 493, 503, 518, 461, 473, 532, 485, 487, 525, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[492, 531, 514, 474, 491, 501, 503, 488, 519, 498, 531, 553, 519, 525, 524, 536, 507, 500, 459, 485, 542, 497, 503, 505, 528, 516, 521, 523, 506, 472, 524, 529, 526, 509, 523, 483, 497, 486, 522, 481, 539, 548, 525, 509, 499, 491, 580, 508, 546, 500, 528, 534, 526, 540, 500, 533, 511, 507, 473, 511, 508, 491, 494, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[538, 564, 511, 458, 486, 532, 487, 532, 544, 549, 539, 448, 511, 523, 510, 472, 520, 512, 521, 504, 548, 520, 532, 534, 498, 512, 540, 451, 526, 527, 530, 503, 513, 544, 499, 524, 516, 521, 503, 504, 493, 500, 545, 470, 505, 471, 502, 530, 470, 480, 513, 533, 513, 486, 533, 486, 508, 500, 517, 518, 533, 523, 519, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 516, 512, 485, 474, 480, 521, 510, 7, 6, 0, 0, 0, 0, 0, 0, 528, 519, 507, 510, 535, 521, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 547, 528, 526, 499, 517, 493, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 507, 544, 515, 553, 499, 518, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 518, 502, 541, 496, 504, 507, 19, 0, 0, 0, 0, 0, 0, 0, 507, 508, 458, 483, 496, 504, 505, 507, 5, 2, 0, 0, 0, 0, 0, 0, 502, 529, 532, 540, 515, 474, 480, 568, 0, 0, 0, 0, 0, 0, 0, 0, 482, 494, 523, 502, 554, 532, 510, 508, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4123, 4143, 4162, 4079, 3975, 4140, 4115] avg=4096.0000 max=4162 max_rank=3 min=3975 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 513, 532, 500, 531, 531, 572, 525, 7, 6, 0, 0, 0, 0, 0, 0, 486, 543, 523, 531, 534, 535, 489, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 551, 515, 539, 536, 540, 470, 496, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 510, 570, 515, 509, 509, 557, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 513, 492, 537, 522, 456, 486, 20, 0, 0, 0, 0, 0, 0, 0, 452, 485, 481, 491, 487, 465, 525, 507, 4, 2, 0, 0, 0, 0, 0, 0, 471, 506, 487, 517, 531, 526, 534, 532, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 497, 496, 502, 509, 489, 483, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4146, 4138, 4196, 4062, 3899, 4104, 4021] avg=4096.0000 max=4202 max_rank=0 min=3899 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16086 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 469, 519, 513, 517, 479, 561, 512, 8, 5, 0, 0, 0, 0, 0, 0, 523, 520, 500, 502, 489, 485, 535, 526, 0, 0, 0, 0, 0, 0, 0, 0, 471, 542, 485, 530, 476, 498, 527, 526, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 522, 509, 494, 521, 502, 548, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 517, 519, 501, 527, 498, 531, 20, 0, 0, 0, 0, 0, 0, 0, 544, 505, 512, 518, 522, 490, 464, 525, 4, 2, 0, 0, 0, 0, 0, 0, 455, 526, 509, 531, 526, 528, 480, 543, 0, 0, 0, 0, 0, 0, 0, 0, 546, 501, 503, 517, 546, 518, 498, 491, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4080, 4055, 4107, 4126, 4086, 4098, 4130] avg=4096.0000 max=4130 max_rank=7 min=4055 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[479, 506, 507, 535, 486, 500, 505, 509, 7, 5, 0, 0, 0, 0, 0, 0, 556, 524, 504, 504, 466, 554, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 489, 530, 509, 493, 537, 492, 514, 476, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 524, 502, 500, 495, 534, 470, 0, 0, 0, 0, 0, 0, 0, 0, 523, 500, 500, 503, 511, 504, 505, 526, 19, 0, 0, 0, 0, 0, 0, 0, 497, 556, 532, 505, 518, 492, 553, 502, 5, 2, 0, 0, 0, 0, 0, 0, 486, 488, 518, 546, 497, 517, 518, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 508, 543, 481, 536, 511, 541, 503, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4149, 4040, 4043, 4091, 4162, 4081, 4163] avg=4096.0000 max=4163 max_rank=7 min=4039 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[540, 495, 526, 547, 489, 499, 491, 511, 7, 5, 0, 0, 0, 0, 0, 0, 498, 489, 507, 514, 461, 485, 489, 513, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 509, 516, 532, 524, 501, 536, 0, 0, 0, 0, 0, 0, 0, 0, 518, 544, 478, 524, 472, 501, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 508, 497, 533, 509, 511, 525, 20, 0, 0, 0, 0, 0, 0, 0, 502, 519, 535, 546, 493, 547, 540, 512, 4, 2, 0, 0, 0, 0, 0, 0, 514, 517, 480, 496, 503, 489, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 545, 493, 561, 484, 517, 497, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 3956, 4110, 4067, 4134, 4200, 4031, 4160] avg=4096.0000 max=4200 max_rank=5 min=3956 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[492, 531, 514, 474, 491, 501, 503, 488, 7, 5, 0, 0, 0, 0, 0, 0, 519, 498, 531, 553, 519, 525, 524, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 459, 485, 540, 497, 503, 505, 0, 0, 0, 0, 0, 0, 0, 0, 528, 516, 521, 523, 491, 472, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 523, 483, 497, 486, 522, 481, 20, 0, 0, 0, 0, 0, 0, 0, 539, 548, 525, 509, 499, 491, 580, 508, 4, 2, 0, 0, 0, 0, 0, 0, 546, 500, 528, 530, 526, 540, 500, 533, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 473, 511, 508, 491, 494, 524, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4178, 3996, 4104, 4047, 4205, 4203, 4029] avg=4096.0000 max=4205 max_rank=5 min=3996 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 522, 561, 498, 515, 487, 528, 527, 12, 10, 0, 0, 0, 0, 0, 0, 478, 536, 499, 517, 478, 527, 530, 498, 0, 0, 0, 0, 0, 0, 0, 0, 470, 524, 492, 536, 530, 530, 508, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 501, 510, 527, 496, 494, 464, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 508, 526, 520, 491, 491, 490, 24, 0, 0, 0, 0, 0, 0, 0, 502, 551, 541, 539, 520, 515, 526, 510, 9, 6, 0, 0, 0, 0, 0, 0, 509, 525, 521, 487, 529, 534, 493, 503, 0, 0, 0, 0, 0, 0, 0, 0, 518, 461, 473, 532, 485, 487, 525, 506, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4063, 4097, 4018, 4088, 4219, 4101, 4008] avg=4096.0000 max=4219 max_rank=5 min=4008 min_rank=7 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[538, 564, 511, 458, 486, 532, 487, 532, 6, 6, 0, 0, 0, 0, 0, 0, 544, 549, 539, 448, 511, 523, 510, 449, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 521, 504, 546, 520, 532, 534, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 540, 451, 510, 527, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 544, 499, 524, 516, 521, 503, 504, 17, 0, 0, 0, 0, 0, 0, 0, 493, 500, 545, 470, 505, 471, 502, 530, 4, 2, 0, 0, 0, 0, 0, 0, 470, 480, 513, 529, 513, 486, 533, 486, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 517, 518, 533, 523, 519, 514, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4073, 4189, 4071, 4141, 4022, 4010, 4142] avg=4096.0000 max=4189 max_rank=2 min=4010 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[516, 513, 483, 518, 525, 536, 465, 527, 497, 477, 531, 537, 503, 539, 530, 492, 511, 488, 521, 520, 520, 515, 545, 543, 498, 513, 498, 500, 497, 514, 495, 528, 505, 525, 526, 501, 496, 535, 513, 531, 509, 545, 503, 531, 502, 521, 494, 497, 535, 503, 536, 490, 511, 500, 519, 495, 508, 506, 512, 492, 507, 528, 471, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[491, 546, 533, 512, 480, 531, 524, 534, 485, 484, 517, 526, 478, 490, 534, 504, 496, 501, 551, 532, 523, 517, 498, 514, 491, 536, 477, 509, 521, 489, 546, 494, 496, 503, 513, 509, 527, 484, 529, 535, 506, 536, 482, 493, 507, 500, 529, 542, 519, 513, 489, 519, 495, 516, 513, 478, 527, 530, 522, 521, 509, 511, 543, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[493, 542, 491, 522, 451, 514, 526, 493, 482, 535, 516, 535, 531, 498, 506, 514, 511, 497, 547, 528, 493, 532, 502, 542, 458, 507, 492, 476, 527, 529, 495, 512, 518, 516, 527, 478, 520, 528, 488, 488, 488, 494, 522, 555, 555, 523, 483, 514, 492, 503, 552, 496, 499, 537, 494, 527, 549, 532, 501, 531, 539, 518, 510, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[524, 500, 491, 480, 498, 508, 469, 510, 525, 541, 505, 515, 532, 525, 509, 480, 467, 498, 542, 537, 534, 496, 488, 531, 511, 514, 492, 560, 492, 521, 507, 528, 546, 489, 480, 530, 515, 556, 512, 516, 494, 522, 497, 528, 526, 505, 483, 502, 529, 495, 522, 525, 497, 502, 519, 528, 500, 554, 512, 532, 487, 509, 526, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[501, 510, 501, 526, 498, 498, 553, 511, 504, 533, 523, 494, 565, 490, 504, 500, 519, 523, 526, 532, 458, 499, 516, 481, 496, 523, 523, 554, 500, 532, 482, 503, 522, 491, 500, 494, 514, 525, 521, 527, 492, 552, 509, 499, 527, 503, 509, 524, 542, 502, 527, 495, 480, 545, 496, 500, 524, 510, 560, 482, 516, 481, 500, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[494, 503, 486, 521, 497, 502, 519, 515, 473, 512, 503, 519, 501, 539, 517, 480, 529, 501, 480, 549, 514, 461, 507, 522, 520, 513, 538, 561, 493, 542, 528, 512, 512, 476, 494, 528, 508, 539, 495, 480, 503, 524, 494, 514, 499, 518, 548, 478, 541, 510, 509, 525, 497, 535, 528, 536, 521, 534, 519, 505, 532, 480, 542, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[468, 506, 510, 541, 525, 497, 484, 538, 485, 508, 555, 519, 499, 542, 502, 535, 512, 508, 489, 506, 464, 507, 485, 523, 480, 475, 497, 542, 484, 482, 530, 490, 552, 516, 533, 529, 477, 522, 539, 483, 513, 525, 497, 504, 493, 533, 531, 494, 538, 505, 542, 525, 487, 499, 545, 515, 533, 511, 544, 552, 517, 519, 495, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[495, 479, 502, 535, 487, 513, 482, 488, 535, 503, 523, 513, 501, 515, 513, 494, 475, 498, 512, 518, 517, 519, 504, 532, 496, 568, 501, 541, 513, 484, 517, 504, 516, 516, 516, 537, 562, 536, 484, 519, 517, 486, 515, 503, 486, 490, 552, 502, 526, 527, 530, 474, 524, 529, 509, 525, 567, 516, 481, 526, 526, 490, 502, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 542, 491, 522, 451, 514, 526, 493, 33, 8, 0, 0, 0, 0, 0, 0, 482, 535, 516, 535, 531, 498, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 547, 528, 493, 532, 502, 542, 4, 0, 0, 0, 0, 0, 0, 0, 458, 507, 492, 476, 527, 529, 495, 512, 8, 5, 0, 0, 0, 0, 0, 0, 518, 516, 527, 478, 520, 514, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 522, 555, 555, 523, 483, 514, 5, 0, 0, 0, 0, 0, 0, 0, 476, 503, 552, 496, 499, 537, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 532, 501, 531, 539, 518, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4117, 4156, 4009, 4049, 4139, 4084, 4141] avg=4096.0000 max=4156 max_rank=2 min=4009 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 546, 533, 512, 480, 531, 524, 534, 32, 9, 0, 0, 0, 0, 0, 0, 485, 484, 517, 526, 478, 490, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 551, 532, 523, 517, 498, 514, 4, 0, 0, 0, 0, 0, 0, 0, 491, 536, 477, 509, 521, 489, 546, 494, 8, 5, 0, 0, 0, 0, 0, 0, 496, 503, 513, 509, 527, 470, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 506, 536, 482, 493, 507, 500, 529, 542, 5, 0, 0, 0, 0, 0, 0, 0, 502, 513, 489, 519, 495, 516, 513, 478, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 522, 521, 509, 511, 543, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4018, 4136, 4076, 4082, 4100, 4025, 4139] avg=4096.0000 max=4192 max_rank=0 min=4018 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 513, 483, 518, 525, 536, 465, 527, 31, 9, 0, 0, 0, 0, 0, 0, 497, 477, 531, 537, 503, 539, 530, 492, 0, 0, 0, 0, 0, 0, 0, 0, 511, 488, 521, 520, 520, 515, 545, 543, 4, 0, 0, 0, 0, 0, 0, 0, 498, 513, 498, 500, 497, 514, 495, 528, 8, 5, 0, 0, 0, 0, 0, 0, 505, 525, 526, 501, 496, 521, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 509, 545, 503, 531, 502, 521, 494, 497, 5, 0, 0, 0, 0, 0, 0, 0, 518, 503, 536, 490, 511, 500, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 477, 506, 512, 492, 507, 528, 471, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4106, 4167, 4056, 4118, 4107, 4072, 4019] avg=4096.0000 max=4167 max_rank=2 min=4019 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[501, 510, 501, 526, 498, 498, 553, 511, 32, 9, 0, 0, 0, 0, 0, 0, 504, 533, 523, 494, 565, 490, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 526, 532, 458, 499, 516, 481, 4, 0, 0, 0, 0, 0, 0, 0, 496, 523, 523, 554, 500, 532, 482, 503, 9, 5, 0, 0, 0, 0, 0, 0, 522, 491, 500, 494, 514, 511, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 492, 552, 509, 499, 527, 503, 509, 524, 5, 0, 0, 0, 0, 0, 0, 0, 524, 502, 527, 495, 480, 545, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 560, 482, 516, 481, 500, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4113, 4058, 4127, 4080, 4120, 4069, 4062] avg=4096.0000 max=4139 max_rank=0 min=4058 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[524, 500, 491, 480, 498, 508, 469, 510, 30, 9, 0, 0, 0, 0, 0, 0, 525, 541, 505, 515, 532, 525, 509, 480, 0, 0, 0, 0, 0, 0, 0, 0, 467, 498, 542, 537, 534, 496, 488, 531, 5, 0, 0, 0, 0, 0, 0, 0, 511, 514, 492, 560, 492, 521, 507, 528, 8, 6, 0, 0, 0, 0, 0, 0, 546, 489, 480, 530, 515, 539, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 494, 522, 497, 528, 526, 505, 483, 502, 6, 0, 0, 0, 0, 0, 0, 0, 512, 495, 522, 525, 497, 502, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 470, 554, 512, 532, 487, 509, 526, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4132, 4098, 4139, 4127, 4063, 4100, 4090] avg=4096.0000 max=4139 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[494, 503, 486, 521, 497, 502, 519, 515, 32, 9, 0, 0, 0, 0, 0, 0, 473, 512, 503, 519, 501, 539, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 529, 501, 480, 549, 514, 461, 507, 522, 4, 0, 0, 0, 0, 0, 0, 0, 520, 513, 538, 561, 493, 542, 528, 512, 9, 5, 0, 0, 0, 0, 0, 0, 512, 476, 494, 528, 508, 525, 495, 480, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 494, 514, 499, 518, 548, 478, 5, 0, 0, 0, 0, 0, 0, 0, 523, 510, 509, 525, 497, 535, 528, 536, 0, 0, 0, 0, 0, 0, 0, 0, 489, 534, 519, 505, 532, 480, 542, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4044, 4067, 4221, 4018, 4083, 4163, 4094] avg=4096.0000 max=4221 max_rank=3 min=4018 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[468, 506, 510, 541, 525, 497, 484, 538, 36, 15, 0, 0, 0, 0, 0, 0, 485, 508, 551, 519, 499, 542, 502, 535, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 489, 506, 464, 507, 485, 523, 9, 4, 0, 0, 0, 0, 0, 0, 480, 475, 497, 542, 484, 482, 530, 490, 13, 10, 0, 0, 0, 0, 0, 0, 552, 516, 533, 529, 477, 493, 539, 483, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 497, 504, 493, 533, 531, 494, 10, 0, 0, 0, 0, 0, 0, 0, 510, 505, 542, 525, 487, 499, 545, 515, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 544, 552, 517, 519, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4141, 4007, 4003, 4122, 4100, 4128, 4147] avg=4096.0000 max=4147 max_rank=7 min=4003 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[495, 479, 502, 535, 487, 513, 482, 488, 34, 9, 0, 0, 0, 0, 0, 0, 535, 503, 523, 513, 501, 515, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 475, 498, 512, 518, 517, 519, 504, 532, 4, 0, 0, 0, 0, 0, 0, 0, 496, 568, 501, 541, 513, 484, 517, 504, 8, 5, 0, 0, 0, 0, 0, 0, 516, 516, 516, 537, 562, 522, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 517, 486, 515, 503, 486, 490, 552, 502, 5, 0, 0, 0, 0, 0, 0, 0, 509, 527, 530, 474, 524, 529, 509, 525, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 481, 526, 526, 490, 502, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4097, 4079, 4137, 4172, 4056, 4127, 4076] avg=4096.0000 max=4172 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[514, 516, 552, 506, 511, 508, 532, 485, 517, 514, 514, 512, 507, 507, 519, 536, 537, 537, 524, 506, 500, 502, 516, 485, 503, 513, 513, 512, 496, 482, 507, 521, 484, 539, 474, 522, 506, 496, 519, 500, 508, 517, 545, 486, 514, 536, 519, 517, 507, 517, 484, 530, 470, 506, 488, 549, 521, 504, 547, 485, 528, 547, 496, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[544, 500, 500, 490, 494, 515, 516, 481, 497, 515, 524, 512, 551, 506, 516, 503, 540, 545, 483, 492, 508, 472, 552, 499, 503, 532, 522, 521, 508, 537, 488, 544, 529, 556, 528, 482, 517, 474, 519, 530, 513, 487, 488, 539, 484, 495, 512, 537, 499, 483, 508, 497, 533, 523, 495, 525, 480, 494, 510, 539, 515, 527, 533, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[518, 539, 551, 488, 544, 506, 492, 513, 523, 541, 490, 489, 508, 512, 502, 474, 538, 547, 498, 516, 519, 517, 553, 489, 525, 518, 495, 514, 538, 508, 490, 509, 546, 526, 498, 482, 535, 554, 513, 534, 500, 492, 505, 500, 501, 485, 482, 521, 494, 525, 486, 503, 532, 492, 529, 519, 481, 522, 531, 507, 494, 524, 471, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[571, 522, 540, 531, 533, 492, 495, 546, 537, 538, 536, 529, 503, 509, 518, 501, 537, 516, 515, 497, 491, 514, 491, 513, 496, 519, 488, 482, 518, 500, 501, 500, 486, 504, 512, 531, 499, 475, 502, 528, 514, 502, 514, 514, 497, 534, 513, 487, 513, 511, 501, 472, 540, 522, 511, 487, 518, 508, 514, 502, 517, 534, 487, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[509, 489, 537, 532, 524, 526, 514, 520, 509, 497, 531, 501, 530, 568, 516, 497, 533, 519, 520, 504, 481, 495, 512, 507, 497, 493, 510, 522, 530, 503, 475, 534, 499, 483, 535, 493, 522, 478, 543, 523, 509, 506, 487, 507, 528, 511, 516, 516, 499, 525, 500, 492, 507, 538, 526, 519, 549, 513, 506, 482, 513, 504, 498, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[517, 527, 512, 550, 510, 521, 505, 494, 543, 492, 490, 503, 475, 529, 500, 505, 481, 525, 564, 508, 517, 489, 530, 475, 490, 496, 517, 546, 505, 539, 545, 527, 488, 510, 541, 529, 516, 498, 516, 503, 504, 502, 511, 481, 489, 508, 494, 524, 480, 502, 532, 513, 546, 535, 485, 529, 489, 496, 491, 542, 541, 533, 490, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[553, 517, 534, 528, 512, 530, 507, 490, 502, 555, 515, 506, 533, 527, 535, 476, 481, 510, 560, 552, 516, 521, 534, 530, 530, 502, 485, 492, 536, 498, 488, 547, 503, 533, 496, 466, 503, 514, 518, 495, 524, 491, 504, 492, 524, 525, 507, 479, 493, 505, 521, 499, 517, 477, 492, 493, 551, 487, 473, 528, 537, 506, 523, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[519, 470, 542, 537, 531, 500, 485, 480, 487, 510, 500, 505, 511, 484, 495, 509, 497, 483, 517, 505, 523, 502, 518, 513, 517, 561, 504, 519, 480, 502, 540, 512, 494, 534, 490, 501, 530, 550, 527, 514, 522, 544, 497, 512, 446, 526, 521, 539, 517, 505, 523, 504, 505, 544, 505, 533, 532, 499, 544, 478, 503, 485, 545, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[544, 500, 466, 490, 494, 515, 516, 481, 0, 0, 0, 0, 0, 0, 0, 0, 497, 513, 524, 512, 551, 506, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 540, 545, 483, 492, 508, 472, 539, 499, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 522, 521, 508, 537, 488, 544, 2, 0, 0, 0, 0, 0, 0, 0, 529, 556, 528, 482, 517, 474, 519, 530, 2, 0, 0, 0, 0, 0, 0, 0, 513, 487, 488, 539, 484, 495, 512, 537, 30, 0, 0, 0, 0, 0, 0, 0, 499, 483, 508, 497, 533, 523, 495, 525, 13, 4, 2, 0, 0, 0, 0, 0, 480, 494, 510, 539, 515, 523, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4122, 4078, 4157, 4137, 4085, 4082, 4101] avg=4096.0000 max=4157 max_rank=3 min=4006 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 514, 506, 511, 508, 532, 485, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 514, 512, 507, 507, 519, 536, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 524, 506, 500, 502, 504, 485, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 513, 512, 496, 482, 507, 521, 2, 0, 0, 0, 0, 0, 0, 0, 484, 539, 474, 522, 506, 496, 519, 500, 2, 0, 0, 0, 0, 0, 0, 0, 508, 517, 545, 486, 514, 536, 519, 517, 34, 0, 0, 0, 0, 0, 0, 0, 507, 517, 484, 530, 470, 506, 488, 549, 12, 4, 2, 0, 0, 0, 0, 0, 521, 504, 547, 485, 528, 543, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4124, 4095, 4049, 4042, 4176, 4069, 4127] avg=4096.0000 max=4176 max_rank=5 min=4042 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 539, 513, 488, 544, 506, 492, 513, 0, 0, 0, 0, 0, 0, 0, 0, 523, 539, 490, 489, 508, 512, 502, 474, 0, 0, 0, 0, 0, 0, 0, 0, 538, 547, 498, 516, 519, 517, 540, 489, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 495, 514, 538, 508, 490, 509, 2, 0, 0, 0, 0, 0, 0, 0, 546, 526, 498, 482, 535, 554, 513, 534, 2, 0, 0, 0, 0, 0, 0, 0, 500, 492, 505, 500, 501, 485, 482, 521, 34, 0, 0, 0, 0, 0, 0, 0, 494, 525, 486, 503, 532, 492, 529, 519, 13, 4, 2, 0, 0, 0, 0, 0, 481, 522, 531, 507, 494, 520, 471, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4037, 4164, 4099, 4190, 4020, 4099, 4046] avg=4096.0000 max=4190 max_rank=4 min=4020 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 527, 477, 550, 510, 521, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 490, 503, 475, 529, 500, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 564, 508, 517, 489, 518, 475, 0, 0, 0, 0, 0, 0, 0, 0, 490, 496, 517, 546, 505, 539, 545, 527, 2, 0, 0, 0, 0, 0, 0, 0, 488, 510, 541, 529, 516, 498, 516, 503, 2, 0, 0, 0, 0, 0, 0, 0, 504, 502, 511, 481, 489, 508, 494, 524, 31, 0, 0, 0, 0, 0, 0, 0, 480, 502, 532, 513, 546, 535, 485, 529, 12, 4, 2, 0, 0, 0, 0, 0, 489, 496, 491, 542, 541, 529, 490, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4035, 4077, 4167, 4103, 4044, 4140, 4101] avg=4096.0000 max=4167 max_rank=3 min=4035 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[571, 522, 497, 531, 533, 492, 495, 546, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 536, 529, 503, 509, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 515, 497, 491, 514, 475, 513, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 488, 482, 518, 500, 501, 500, 8, 0, 0, 0, 0, 0, 0, 0, 486, 504, 512, 531, 499, 475, 502, 528, 4, 5, 0, 0, 0, 0, 0, 0, 514, 502, 514, 514, 497, 534, 513, 487, 36, 0, 0, 0, 0, 0, 0, 0, 513, 511, 501, 472, 540, 522, 511, 487, 16, 7, 4, 0, 0, 0, 0, 0, 518, 508, 514, 502, 517, 526, 487, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4158, 4058, 4012, 4046, 4111, 4084, 4112] avg=4096.0000 max=4187 max_rank=0 min=4012 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 489, 500, 532, 524, 526, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 531, 501, 530, 568, 516, 497, 0, 0, 0, 0, 0, 0, 0, 0, 533, 519, 520, 504, 481, 495, 500, 507, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 510, 522, 530, 503, 475, 534, 2, 0, 0, 0, 0, 0, 0, 0, 499, 483, 535, 493, 522, 478, 543, 523, 2, 0, 0, 0, 0, 0, 0, 0, 509, 506, 487, 507, 528, 511, 516, 516, 33, 0, 0, 0, 0, 0, 0, 0, 499, 525, 500, 492, 507, 538, 526, 519, 12, 4, 2, 0, 0, 0, 0, 0, 549, 513, 506, 482, 513, 500, 498, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4147, 4059, 4066, 4078, 4113, 4124, 4067] avg=4096.0000 max=4147 max_rank=1 min=4059 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 517, 497, 528, 512, 530, 507, 490, 0, 0, 0, 0, 0, 0, 0, 0, 502, 552, 515, 506, 533, 527, 535, 476, 0, 0, 0, 0, 0, 0, 0, 0, 481, 510, 560, 552, 516, 521, 521, 530, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 485, 492, 536, 498, 488, 547, 3, 0, 0, 0, 0, 0, 0, 0, 503, 533, 496, 466, 503, 514, 518, 495, 2, 0, 0, 0, 0, 0, 0, 0, 524, 491, 504, 492, 524, 525, 507, 479, 33, 0, 0, 0, 0, 0, 0, 0, 493, 505, 521, 499, 517, 477, 492, 493, 13, 4, 2, 0, 0, 0, 0, 0, 551, 487, 473, 528, 537, 502, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4146, 4191, 4081, 4030, 4079, 4016, 4091] avg=4096.0000 max=4191 max_rank=2 min=4016 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 470, 505, 537, 531, 500, 485, 480, 0, 0, 0, 0, 0, 0, 0, 0, 487, 508, 500, 505, 511, 484, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 517, 505, 523, 502, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 561, 504, 519, 480, 502, 540, 512, 2, 0, 0, 0, 0, 0, 0, 0, 494, 534, 490, 501, 530, 550, 527, 514, 2, 0, 0, 0, 0, 0, 0, 0, 522, 544, 497, 512, 446, 526, 521, 539, 33, 0, 0, 0, 0, 0, 0, 0, 517, 505, 523, 504, 505, 544, 505, 533, 12, 4, 2, 0, 0, 0, 0, 0, 532, 499, 544, 478, 503, 481, 545, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 3999, 4046, 4137, 4142, 4140, 4154, 4123] avg=4096.0000 max=4154 max_rank=6 min=3999 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[520, 521, 539, 522, 545, 519, 502, 498, 492, 523, 484, 519, 506, 505, 505, 547, 541, 508, 496, 535, 543, 511, 505, 554, 518, 529, 514, 511, 460, 512, 513, 511, 513, 480, 495, 467, 554, 515, 494, 547, 515, 526, 521, 511, 520, 476, 492, 515, 539, 487, 490, 525, 487, 538, 580, 455, 496, 483, 515, 531, 511, 495, 477, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[549, 553, 471, 536, 485, 506, 490, 504, 472, 482, 543, 505, 562, 492, 514, 531, 501, 520, 539, 494, 516, 484, 506, 537, 554, 504, 529, 521, 481, 524, 518, 501, 502, 524, 547, 521, 535, 546, 500, 567, 467, 515, 500, 498, 489, 483, 520, 475, 507, 544, 485, 482, 540, 531, 552, 542, 505, 493, 508, 492, 485, 496, 515, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[499, 541, 505, 536, 522, 505, 537, 517, 530, 508, 559, 487, 503, 505, 499, 511, 491, 485, 499, 535, 537, 520, 483, 553, 508, 470, 529, 528, 537, 482, 487, 514, 494, 496, 486, 534, 504, 504, 502, 512, 497, 540, 533, 456, 547, 496, 527, 550, 520, 487, 508, 550, 512, 523, 553, 508, 496, 479, 509, 516, 485, 499, 488, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[560, 481, 502, 487, 542, 512, 519, 515, 545, 543, 526, 509, 492, 486, 539, 514, 497, 498, 537, 473, 481, 504, 507, 479, 472, 493, 506, 555, 527, 526, 503, 504, 506, 523, 527, 553, 483, 518, 567, 488, 523, 523, 507, 522, 502, 505, 494, 518, 512, 530, 483, 546, 518, 501, 517, 514, 543, 509, 499, 489, 497, 515, 499, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[554, 527, 484, 524, 575, 522, 528, 513, 501, 511, 496, 477, 515, 508, 465, 502, 488, 488, 474, 474, 529, 523, 481, 531, 547, 512, 505, 517, 518, 491, 525, 526, 542, 553, 466, 519, 520, 502, 548, 469, 504, 524, 479, 479, 497, 514, 528, 498, 526, 508, 509, 541, 521, 535, 509, 530, 518, 524, 501, 501, 529, 502, 521, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[544, 552, 503, 507, 514, 505, 491, 505, 539, 534, 503, 527, 513, 501, 504, 488, 569, 502, 529, 501, 488, 521, 493, 511, 513, 497, 509, 516, 534, 501, 521, 484, 505, 505, 524, 513, 491, 501, 481, 500, 531, 536, 523, 503, 524, 473, 508, 502, 524, 531, 509, 529, 482, 503, 471, 551, 492, 506, 470, 547, 541, 520, 515, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[499, 497, 492, 521, 488, 507, 491, 492, 539, 515, 533, 521, 503, 516, 514, 482, 479, 508, 546, 497, 512, 517, 510, 506, 532, 498, 500, 534, 517, 519, 499, 489, 533, 526, 524, 554, 486, 494, 503, 492, 510, 507, 556, 472, 514, 535, 476, 498, 527, 529, 523, 520, 539, 533, 512, 489, 536, 532, 531, 503, 520, 487, 531, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[514, 509, 545, 509, 530, 506, 541, 522, 527, 504, 525, 486, 509, 472, 516, 539, 494, 541, 517, 502, 523, 523, 520, 501, 500, 498, 486, 501, 524, 491, 506, 491, 526, 539, 527, 472, 525, 525, 496, 501, 498, 498, 518, 505, 559, 525, 501, 504, 515, 530, 534, 509, 546, 496, 516, 510, 504, 503, 513, 476, 485, 522, 507, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[487, 521, 539, 522, 545, 519, 502, 498, 0, 0, 0, 0, 0, 0, 0, 0, 492, 523, 484, 519, 506, 505, 505, 547, 5, 0, 0, 0, 0, 0, 0, 0, 541, 508, 496, 535, 543, 511, 505, 554, 6, 5, 0, 0, 0, 0, 0, 0, 518, 529, 514, 511, 460, 512, 513, 511, 15, 0, 0, 0, 0, 0, 0, 0, 513, 469, 495, 467, 554, 515, 494, 547, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 521, 511, 520, 476, 492, 515, 13, 13, 0, 0, 0, 0, 0, 0, 539, 487, 490, 525, 487, 538, 525, 455, 0, 0, 0, 0, 0, 0, 0, 0, 496, 483, 515, 531, 511, 495, 477, 510, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4086, 4204, 4083, 4054, 4102, 4046, 4060] avg=4096.0000 max=4204 max_rank=2 min=4046 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 553, 471, 536, 485, 506, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 472, 482, 543, 505, 562, 492, 514, 531, 5, 0, 0, 0, 0, 0, 0, 0, 501, 520, 539, 494, 516, 484, 506, 537, 6, 6, 0, 0, 0, 0, 0, 0, 554, 504, 529, 521, 481, 524, 518, 501, 16, 0, 0, 0, 0, 0, 0, 0, 502, 513, 547, 521, 535, 546, 500, 567, 0, 0, 0, 0, 0, 0, 0, 0, 467, 515, 500, 498, 489, 483, 520, 475, 14, 12, 0, 0, 0, 0, 0, 0, 507, 544, 485, 482, 540, 531, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 508, 492, 485, 496, 515, 478, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4106, 4109, 4148, 4231, 3973, 4131, 4012] avg=4096.0000 max=4231 max_rank=4 min=3973 min_rank=5 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 527, 484, 524, 575, 522, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 511, 496, 477, 515, 508, 465, 502, 6, 0, 0, 0, 0, 0, 0, 0, 488, 488, 474, 474, 529, 523, 481, 531, 7, 6, 0, 0, 0, 0, 0, 0, 547, 512, 505, 517, 518, 491, 525, 526, 16, 0, 0, 0, 0, 0, 0, 0, 542, 540, 466, 519, 520, 502, 548, 469, 0, 0, 0, 0, 0, 0, 0, 0, 504, 524, 479, 479, 497, 514, 528, 498, 14, 11, 0, 0, 0, 0, 0, 0, 526, 508, 509, 541, 521, 535, 461, 530, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 501, 501, 529, 502, 521, 520, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 3981, 4001, 4157, 4106, 4048, 4131, 4153] avg=4096.0000 max=4191 max_rank=0 min=3981 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[468, 541, 505, 536, 522, 505, 537, 517, 0, 0, 0, 0, 0, 0, 0, 0, 530, 508, 559, 487, 503, 505, 499, 511, 5, 0, 0, 0, 0, 0, 0, 0, 491, 485, 499, 535, 537, 520, 483, 553, 6, 5, 0, 0, 0, 0, 0, 0, 508, 470, 529, 528, 537, 482, 487, 514, 14, 0, 0, 0, 0, 0, 0, 0, 494, 485, 486, 534, 504, 504, 502, 512, 0, 0, 0, 0, 0, 0, 0, 0, 497, 540, 533, 456, 547, 496, 527, 550, 12, 12, 0, 0, 0, 0, 0, 0, 520, 487, 508, 550, 512, 523, 501, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 479, 509, 516, 485, 499, 488, 535, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4107, 4114, 4069, 4021, 4170, 4109, 4047] avg=4096.0000 max=4170 max_rank=5 min=4021 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 481, 502, 487, 542, 512, 519, 515, 0, 0, 0, 0, 0, 0, 0, 0, 545, 543, 526, 509, 492, 486, 539, 514, 5, 0, 0, 0, 0, 0, 0, 0, 497, 498, 537, 473, 481, 504, 507, 479, 6, 6, 0, 0, 0, 0, 0, 0, 472, 493, 506, 555, 527, 526, 503, 504, 16, 0, 0, 0, 0, 0, 0, 0, 506, 512, 527, 553, 483, 518, 567, 488, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 507, 522, 502, 505, 494, 518, 14, 11, 0, 0, 0, 0, 0, 0, 512, 530, 483, 546, 518, 501, 469, 514, 0, 0, 0, 0, 0, 0, 0, 0, 543, 509, 499, 489, 497, 515, 499, 503, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4159, 3988, 4102, 4154, 4119, 4073, 4091] avg=4096.0000 max=4159 max_rank=1 min=3988 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 552, 503, 507, 514, 505, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 539, 534, 503, 527, 513, 501, 504, 488, 5, 0, 0, 0, 0, 0, 0, 0, 569, 502, 529, 501, 488, 521, 493, 511, 6, 6, 0, 0, 0, 0, 0, 0, 513, 497, 509, 516, 534, 501, 521, 484, 16, 0, 0, 0, 0, 0, 0, 0, 505, 494, 524, 513, 491, 501, 481, 500, 0, 0, 0, 0, 0, 0, 0, 0, 531, 536, 523, 503, 524, 473, 508, 502, 13, 10, 0, 0, 0, 0, 0, 0, 524, 531, 509, 529, 482, 503, 427, 551, 0, 0, 0, 0, 0, 0, 0, 0, 492, 506, 470, 547, 541, 520, 515, 538, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4114, 4126, 4091, 4009, 4123, 4056, 4163] avg=4096.0000 max=4163 max_rank=7 min=4009 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[456, 497, 492, 521, 488, 507, 491, 492, 0, 0, 0, 0, 0, 0, 0, 0, 539, 515, 533, 521, 503, 516, 514, 482, 9, 0, 0, 0, 0, 0, 0, 0, 479, 508, 546, 497, 512, 517, 510, 506, 10, 9, 0, 0, 0, 0, 0, 0, 532, 498, 500, 534, 517, 519, 499, 489, 18, 0, 0, 0, 0, 0, 0, 0, 533, 507, 524, 554, 486, 494, 503, 492, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 556, 472, 514, 535, 476, 498, 16, 17, 0, 0, 0, 0, 0, 0, 527, 529, 523, 520, 539, 533, 454, 489, 0, 0, 0, 0, 0, 0, 0, 0, 536, 532, 531, 503, 520, 487, 531, 503, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3944, 4132, 4094, 4106, 4093, 4101, 4114, 4184] avg=4096.0000 max=4184 max_rank=7 min=3944 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[481, 509, 545, 509, 530, 506, 541, 522, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 525, 486, 509, 472, 516, 539, 5, 0, 0, 0, 0, 0, 0, 0, 494, 541, 517, 502, 523, 523, 520, 501, 6, 5, 0, 0, 0, 0, 0, 0, 500, 498, 486, 501, 524, 491, 506, 491, 15, 0, 0, 0, 0, 0, 0, 0, 526, 528, 527, 472, 525, 525, 496, 501, 0, 0, 0, 0, 0, 0, 0, 0, 498, 498, 518, 505, 559, 525, 501, 504, 13, 11, 0, 0, 0, 0, 0, 0, 515, 530, 534, 509, 546, 496, 468, 510, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 513, 476, 485, 522, 507, 511, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4083, 4132, 4012, 4100, 4132, 4108, 4058] avg=4096.0000 max=4143 max_rank=0 min=4012 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 3968, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-26 15:43:13] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 281.0 | throughput per GPU (TFLOP/s/GPU): 439.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.602537E+00 | loss scale: 1.0 | grad norm: 3.930 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[491, 475, 553, 543, 509, 502, 502, 530, 528, 552, 514, 470, 503, 536, 499, 505, 530, 513, 518, 511, 514, 517, 533, 511, 530, 518, 503, 501, 510, 519, 491, 521, 493, 478, 527, 503, 516, 528, 516, 526, 539, 467, 530, 457, 505, 496, 527, 533, 509, 507, 526, 496, 510, 479, 490, 537, 510, 474, 506, 533, 553, 501, 524, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[540, 520, 506, 538, 494, 500, 533, 500, 499, 534, 535, 511, 481, 501, 483, 506, 500, 529, 474, 490, 496, 493, 538, 526, 510, 513, 534, 497, 522, 533, 499, 525, 515, 536, 511, 521, 494, 512, 515, 509, 520, 501, 507, 536, 479, 503, 543, 493, 517, 525, 513, 501, 516, 474, 509, 532, 489, 529, 515, 521, 527, 519, 503, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[520, 498, 518, 527, 512, 488, 532, 489, 492, 509, 526, 549, 525, 498, 495, 523, 515, 522, 558, 481, 492, 554, 479, 529, 528, 457, 509, 529, 544, 517, 537, 556, 492, 494, 540, 499, 508, 496, 518, 498, 528, 523, 472, 524, 518, 457, 503, 482, 504, 527, 495, 512, 522, 460, 536, 516, 528, 488, 515, 502, 544, 529, 505, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[527, 516, 502, 505, 501, 476, 506, 553, 507, 488, 531, 531, 505, 525, 501, 498, 524, 514, 481, 556, 521, 514, 517, 493, 555, 513, 516, 499, 496, 489, 528, 503, 537, 518, 524, 519, 482, 523, 503, 526, 531, 486, 482, 507, 514, 514, 507, 512, 537, 503, 526, 496, 487, 511, 513, 493, 542, 531, 528, 479, 517, 515, 515, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[475, 504, 512, 515, 514, 511, 538, 506, 512, 499, 529, 524, 484, 484, 496, 551, 499, 512, 559, 504, 515, 523, 579, 507, 501, 553, 512, 479, 534, 502, 514, 519, 525, 463, 535, 493, 520, 488, 518, 478, 493, 459, 492, 524, 516, 478, 494, 519, 501, 518, 525, 532, 526, 484, 514, 525, 530, 522, 521, 513, 534, 525, 509, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[505, 497, 515, 515, 557, 488, 463, 480, 519, 509, 515, 500, 530, 519, 486, 554, 481, 528, 480, 509, 502, 518, 528, 524, 513, 530, 517, 549, 436, 512, 518, 513, 525, 503, 498, 497, 538, 481, 537, 489, 503, 533, 510, 479, 550, 476, 585, 504, 512, 488, 521, 509, 489, 547, 543, 525, 527, 507, 494, 502, 502, 532, 509, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[540, 511, 510, 507, 517, 541, 501, 525, 521, 540, 525, 503, 486, 454, 520, 482, 536, 538, 483, 522, 483, 537, 551, 500, 499, 501, 502, 509, 500, 532, 509, 502, 514, 474, 474, 524, 548, 502, 542, 529, 516, 487, 474, 533, 521, 520, 526, 485, 506, 484, 551, 504, 523, 518, 502, 545, 491, 512, 503, 505, 535, 514, 510, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[471, 543, 498, 508, 508, 522, 535, 492, 498, 520, 502, 480, 502, 542, 517, 517, 523, 507, 553, 511, 531, 485, 496, 487, 508, 490, 525, 532, 512, 536, 490, 525, 493, 520, 521, 512, 523, 514, 503, 494, 484, 545, 513, 494, 496, 505, 516, 506, 503, 525, 536, 520, 515, 582, 481, 488, 490, 536, 491, 534, 540, 514, 486, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[491, 475, 553, 543, 509, 502, 502, 530, 4, 0, 0, 0, 0, 0, 0, 0, 528, 552, 514, 470, 503, 536, 499, 505, 9, 1, 0, 0, 0, 0, 0, 0, 530, 513, 518, 511, 514, 517, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 518, 503, 501, 510, 519, 491, 508, 0, 0, 0, 0, 0, 0, 0, 0, 493, 478, 527, 503, 516, 528, 516, 526, 10, 4, 0, 0, 0, 0, 0, 0, 539, 467, 530, 457, 505, 496, 527, 533, 30, 13, 0, 0, 0, 0, 0, 0, 509, 507, 521, 496, 510, 479, 490, 537, 0, 0, 0, 0, 0, 0, 0, 0, 510, 474, 506, 533, 523, 501, 524, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4117, 4124, 4080, 4101, 4097, 4049, 4091] avg=4096.0000 max=4124 max_rank=2 min=4049 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[527, 516, 502, 505, 501, 476, 506, 553, 4, 0, 0, 0, 0, 0, 0, 0, 507, 488, 531, 531, 505, 525, 501, 498, 8, 1, 0, 0, 0, 0, 0, 0, 524, 514, 481, 556, 521, 514, 496, 493, 0, 0, 0, 0, 0, 0, 0, 0, 555, 513, 516, 499, 496, 489, 528, 491, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 524, 519, 482, 523, 503, 526, 9, 4, 0, 0, 0, 0, 0, 0, 531, 486, 482, 507, 514, 514, 507, 512, 28, 12, 0, 0, 0, 0, 0, 0, 537, 503, 521, 496, 487, 511, 513, 493, 0, 0, 0, 0, 0, 0, 0, 0, 542, 531, 528, 479, 489, 515, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4099, 4087, 4145, 4093, 4061, 4098] avg=4096.0000 max=4145 max_rank=4 min=4061 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[540, 520, 506, 538, 494, 500, 533, 500, 4, 0, 0, 0, 0, 0, 0, 0, 499, 534, 535, 511, 481, 501, 483, 506, 9, 1, 0, 0, 0, 0, 0, 0, 500, 529, 474, 490, 496, 493, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 534, 497, 522, 533, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 511, 521, 494, 512, 515, 509, 10, 4, 0, 0, 0, 0, 0, 0, 520, 501, 507, 536, 479, 503, 543, 493, 28, 13, 0, 0, 0, 0, 0, 0, 517, 525, 508, 501, 516, 474, 509, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 529, 515, 521, 499, 519, 503, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4060, 4023, 4120, 4127, 4123, 4082, 4098] avg=4096.0000 max=4135 max_rank=0 min=4023 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 498, 518, 527, 512, 488, 532, 489, 4, 0, 0, 0, 0, 0, 0, 0, 492, 509, 526, 549, 525, 498, 495, 523, 9, 1, 0, 0, 0, 0, 0, 0, 515, 522, 558, 481, 492, 554, 459, 529, 0, 0, 0, 0, 0, 0, 0, 0, 528, 457, 509, 529, 544, 517, 537, 543, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 540, 499, 508, 496, 518, 498, 9, 4, 0, 0, 0, 0, 0, 0, 528, 523, 472, 524, 518, 457, 503, 482, 29, 11, 0, 0, 0, 0, 0, 0, 504, 527, 490, 512, 522, 460, 536, 516, 0, 0, 0, 0, 0, 0, 0, 0, 528, 488, 515, 502, 515, 529, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4127, 4110, 4164, 4058, 4047, 4067, 4107] avg=4096.0000 max=4164 max_rank=3 min=4047 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[475, 504, 512, 515, 514, 511, 538, 506, 4, 0, 0, 0, 0, 0, 0, 0, 512, 499, 529, 524, 484, 484, 496, 551, 9, 1, 0, 0, 0, 0, 0, 0, 499, 512, 559, 504, 515, 523, 554, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 553, 512, 479, 534, 502, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 525, 463, 535, 493, 520, 488, 518, 478, 11, 4, 0, 0, 0, 0, 0, 0, 493, 459, 492, 524, 516, 478, 494, 519, 29, 14, 0, 0, 0, 0, 0, 0, 501, 518, 520, 532, 526, 484, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 530, 522, 521, 513, 505, 525, 509, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4089, 4173, 4101, 4035, 4018, 4120, 4153] avg=4096.0000 max=4173 max_rank=2 min=4018 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[471, 543, 498, 508, 508, 522, 535, 492, 4, 0, 0, 0, 0, 0, 0, 0, 498, 520, 502, 480, 502, 542, 517, 517, 9, 1, 0, 0, 0, 0, 0, 0, 523, 507, 553, 511, 531, 485, 475, 487, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 525, 532, 512, 536, 490, 512, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 521, 512, 523, 514, 503, 494, 9, 4, 0, 0, 0, 0, 0, 0, 484, 545, 513, 494, 496, 505, 516, 506, 29, 12, 0, 0, 0, 0, 0, 0, 503, 525, 531, 520, 515, 582, 481, 488, 0, 0, 0, 0, 0, 0, 0, 0, 490, 536, 491, 534, 511, 514, 486, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4088, 4072, 4105, 4093, 4100, 4145, 4084] avg=4096.0000 max=4145 max_rank=6 min=4072 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 497, 515, 515, 557, 488, 463, 480, 10, 0, 0, 0, 0, 0, 0, 0, 519, 509, 515, 500, 530, 519, 486, 554, 12, 8, 0, 0, 0, 0, 0, 0, 481, 528, 480, 509, 502, 518, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 517, 549, 436, 512, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 498, 497, 538, 481, 537, 489, 13, 7, 0, 0, 0, 0, 0, 0, 503, 533, 510, 479, 550, 476, 585, 504, 29, 17, 0, 0, 0, 0, 0, 0, 512, 488, 503, 509, 489, 547, 543, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 507, 494, 502, 473, 532, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4152, 4040, 4069, 4088, 4186, 4116, 4087] avg=4096.0000 max=4186 max_rank=5 min=4030 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[540, 511, 510, 507, 517, 541, 501, 525, 4, 0, 0, 0, 0, 0, 0, 0, 521, 540, 525, 503, 486, 454, 520, 482, 8, 1, 0, 0, 0, 0, 0, 0, 536, 538, 483, 522, 483, 537, 528, 500, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 502, 509, 500, 532, 509, 490, 0, 0, 0, 0, 0, 0, 0, 0, 514, 474, 474, 524, 548, 502, 542, 529, 10, 4, 0, 0, 0, 0, 0, 0, 516, 487, 474, 533, 521, 520, 526, 485, 29, 13, 0, 0, 0, 0, 0, 0, 506, 484, 546, 504, 523, 518, 502, 545, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 503, 505, 506, 514, 510, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4040, 4127, 4042, 4121, 4104, 4128, 4050] avg=4096.0000 max=4156 max_rank=0 min=4040 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 3968, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[463, 479, 518, 492, 531, 513, 525, 538, 509, 482, 494, 540, 508, 514, 472, 505, 488, 515, 551, 508, 501, 513, 502, 539, 486, 507, 516, 477, 495, 508, 509, 507, 509, 504, 552, 518, 551, 498, 517, 522, 516, 490, 497, 518, 502, 524, 526, 517, 483, 490, 532, 501, 534, 517, 526, 553, 495, 495, 525, 494, 538, 577, 493, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[534, 509, 511, 516, 500, 476, 495, 514, 568, 510, 556, 538, 546, 491, 509, 504, 543, 488, 506, 487, 518, 543, 488, 493, 519, 514, 546, 520, 517, 521, 515, 501, 511, 491, 489, 496, 517, 489, 512, 526, 525, 496, 516, 527, 474, 510, 494, 519, 540, 531, 538, 500, 486, 558, 491, 504, 497, 503, 512, 511, 520, 522, 460, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[497, 474, 529, 483, 541, 543, 494, 522, 505, 504, 521, 500, 463, 503, 471, 566, 515, 522, 491, 527, 500, 511, 485, 502, 513, 531, 511, 541, 506, 512, 560, 504, 502, 558, 543, 509, 488, 517, 477, 514, 513, 521, 501, 536, 493, 506, 474, 506, 542, 510, 511, 521, 530, 504, 542, 522, 513, 508, 505, 523, 504, 543, 478, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[492, 496, 519, 528, 487, 520, 525, 499, 479, 487, 503, 533, 523, 487, 540, 521, 506, 508, 508, 520, 553, 517, 509, 535, 572, 508, 502, 530, 482, 520, 478, 518, 503, 524, 479, 504, 511, 516, 470, 526, 521, 535, 489, 542, 501, 496, 510, 506, 506, 516, 504, 503, 525, 489, 483, 533, 555, 499, 522, 521, 551, 503, 491, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[514, 477, 510, 531, 503, 500, 487, 496, 460, 514, 520, 558, 512, 549, 508, 475, 487, 514, 495, 508, 586, 489, 494, 573, 532, 507, 517, 522, 490, 515, 527, 515, 509, 517, 510, 519, 506, 507, 475, 518, 510, 507, 517, 497, 521, 513, 514, 527, 522, 515, 523, 485, 528, 532, 518, 526, 528, 526, 536, 525, 461, 473, 486, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[489, 529, 466, 519, 515, 509, 524, 510, 524, 469, 537, 569, 528, 559, 484, 519, 482, 533, 509, 510, 505, 489, 464, 549, 501, 536, 521, 546, 507, 492, 524, 527, 490, 503, 481, 534, 508, 506, 533, 532, 518, 503, 517, 517, 510, 507, 490, 511, 470, 491, 465, 495, 516, 500, 538, 533, 543, 487, 537, 522, 498, 522, 551, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[528, 538, 517, 505, 529, 524, 505, 482, 531, 553, 540, 503, 491, 496, 470, 510, 534, 539, 469, 518, 529, 503, 511, 489, 498, 495, 512, 508, 486, 523, 483, 544, 539, 526, 536, 513, 522, 495, 531, 468, 534, 473, 507, 497, 534, 524, 501, 547, 532, 473, 557, 476, 494, 537, 507, 492, 512, 534, 513, 525, 495, 500, 516, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[474, 497, 513, 524, 528, 504, 523, 546, 487, 503, 481, 560, 508, 562, 505, 542, 547, 520, 484, 523, 545, 545, 507, 524, 500, 506, 520, 501, 507, 500, 559, 503, 538, 489, 517, 510, 507, 492, 504, 544, 522, 528, 500, 506, 502, 519, 514, 505, 520, 493, 524, 474, 468, 484, 529, 522, 516, 546, 468, 498, 491, 496, 486, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 474, 529, 483, 541, 543, 494, 522, 12, 10, 0, 0, 0, 0, 0, 0, 505, 504, 521, 488, 463, 503, 471, 566, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 491, 527, 489, 511, 485, 502, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 511, 541, 506, 512, 547, 504, 0, 0, 0, 0, 0, 0, 0, 0, 502, 558, 543, 509, 488, 517, 477, 514, 5, 0, 0, 0, 0, 0, 0, 0, 513, 521, 501, 536, 493, 506, 474, 506, 6, 3, 0, 0, 0, 0, 0, 0, 542, 510, 511, 521, 530, 504, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 513, 508, 505, 523, 504, 543, 478, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4021, 4042, 4165, 4113, 4059, 4182, 4081] avg=4096.0000 max=4182 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[534, 509, 511, 516, 500, 476, 495, 514, 13, 9, 0, 0, 0, 0, 0, 0, 568, 510, 556, 525, 546, 491, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 506, 487, 507, 543, 488, 493, 0, 0, 0, 0, 0, 0, 0, 0, 519, 514, 546, 520, 517, 521, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 511, 491, 489, 496, 517, 489, 512, 526, 5, 0, 0, 0, 0, 0, 0, 0, 525, 496, 516, 527, 474, 510, 494, 519, 6, 2, 0, 0, 0, 0, 0, 0, 540, 531, 538, 500, 486, 558, 491, 504, 0, 0, 0, 0, 0, 0, 0, 0, 497, 503, 512, 511, 520, 522, 460, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4209, 4055, 4142, 4036, 4069, 4148, 4032] avg=4096.0000 max=4209 max_rank=1 min=4032 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[463, 479, 518, 492, 531, 513, 525, 538, 13, 9, 0, 0, 0, 0, 0, 0, 509, 482, 494, 527, 508, 514, 472, 505, 0, 0, 0, 0, 0, 0, 0, 0, 488, 515, 551, 508, 490, 513, 502, 539, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 516, 477, 495, 508, 498, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 504, 552, 518, 551, 498, 517, 522, 5, 0, 0, 0, 0, 0, 0, 0, 516, 490, 497, 518, 502, 524, 526, 517, 6, 2, 0, 0, 0, 0, 0, 0, 483, 490, 532, 501, 534, 517, 526, 553, 0, 0, 0, 0, 0, 0, 0, 0, 495, 495, 525, 494, 538, 577, 493, 549, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4011, 4106, 3994, 4176, 4098, 4136, 4166] avg=4096.0000 max=4176 max_rank=4 min=3994 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 496, 519, 528, 487, 520, 525, 499, 13, 8, 0, 0, 0, 0, 0, 0, 479, 487, 503, 520, 523, 487, 540, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 508, 520, 541, 517, 509, 535, 0, 0, 0, 0, 0, 0, 0, 0, 572, 508, 502, 530, 482, 520, 468, 518, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 479, 504, 511, 516, 470, 526, 6, 0, 0, 0, 0, 0, 0, 0, 521, 535, 489, 542, 501, 496, 510, 506, 6, 2, 0, 0, 0, 0, 0, 0, 506, 516, 504, 503, 525, 489, 483, 533, 0, 0, 0, 0, 0, 0, 0, 0, 555, 499, 522, 521, 551, 503, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4060, 4144, 4100, 4039, 4108, 4059, 4171] avg=4096.0000 max=4171 max_rank=7 min=4039 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 477, 510, 531, 503, 500, 487, 496, 14, 9, 0, 0, 0, 0, 0, 0, 460, 514, 520, 544, 512, 549, 508, 475, 0, 0, 0, 0, 0, 0, 0, 0, 487, 514, 495, 508, 573, 489, 494, 573, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 517, 522, 490, 515, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 510, 519, 506, 507, 475, 518, 6, 0, 0, 0, 0, 0, 0, 0, 510, 507, 517, 497, 521, 513, 514, 527, 7, 3, 0, 0, 0, 0, 0, 0, 522, 515, 523, 485, 528, 532, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 536, 525, 461, 473, 486, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4082, 4133, 4113, 4067, 4116, 4149, 4067] avg=4096.0000 max=4149 max_rank=6 min=4041 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 529, 466, 519, 515, 509, 524, 510, 14, 9, 0, 0, 0, 0, 0, 0, 524, 469, 537, 555, 528, 559, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 482, 533, 509, 510, 494, 489, 464, 549, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 521, 546, 507, 492, 512, 527, 0, 0, 0, 0, 0, 0, 0, 0, 490, 503, 481, 534, 508, 506, 533, 532, 5, 0, 0, 0, 0, 0, 0, 0, 518, 503, 517, 517, 510, 507, 490, 511, 6, 3, 0, 0, 0, 0, 0, 0, 470, 491, 465, 495, 516, 500, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0, 543, 487, 537, 522, 498, 522, 551, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4175, 4030, 4142, 4092, 4082, 4008, 4155] avg=4096.0000 max=4175 max_rank=1 min=4008 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[474, 497, 513, 524, 528, 504, 523, 546, 14, 10, 0, 0, 0, 0, 0, 0, 487, 503, 481, 546, 508, 562, 505, 542, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 484, 523, 534, 545, 507, 524, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 520, 501, 507, 500, 546, 503, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 517, 510, 507, 492, 504, 544, 5, 0, 0, 0, 0, 0, 0, 0, 522, 528, 500, 506, 502, 519, 514, 505, 6, 3, 0, 0, 0, 0, 0, 0, 520, 493, 524, 474, 468, 484, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 516, 546, 468, 498, 491, 496, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4134, 4184, 4083, 4106, 4105, 4014, 4009] avg=4096.0000 max=4184 max_rank=2 min=4009 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[528, 538, 517, 505, 529, 524, 505, 482, 18, 14, 0, 0, 0, 0, 0, 0, 531, 553, 540, 485, 491, 496, 470, 510, 0, 0, 0, 0, 0, 0, 0, 0, 534, 539, 469, 518, 511, 503, 511, 489, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 512, 508, 486, 523, 463, 544, 0, 0, 0, 0, 0, 0, 0, 0, 539, 526, 536, 513, 522, 495, 531, 468, 9, 0, 0, 0, 0, 0, 0, 0, 534, 473, 507, 497, 534, 524, 501, 547, 8, 6, 0, 0, 0, 0, 0, 0, 532, 473, 557, 476, 494, 537, 507, 492, 3, 1, 0, 0, 0, 0, 0, 0, 509, 534, 513, 525, 495, 500, 516, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4076, 4074, 4029, 4139, 4131, 4072, 4087] avg=4096.0000 max=4160 max_rank=0 min=4029 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[494, 500, 495, 501, 535, 509, 516, 512, 525, 495, 511, 525, 495, 494, 527, 461, 530, 519, 547, 511, 524, 511, 513, 546, 507, 512, 492, 481, 518, 505, 554, 534, 511, 497, 520, 469, 512, 506, 513, 548, 517, 508, 511, 523, 539, 493, 536, 513, 538, 514, 500, 544, 502, 520, 495, 471, 558, 453, 506, 534, 527, 500, 486, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[503, 497, 514, 528, 507, 505, 476, 501, 519, 509, 527, 493, 471, 490, 537, 548, 539, 480, 531, 520, 524, 489, 537, 507, 509, 514, 510, 523, 548, 491, 499, 511, 546, 490, 543, 489, 504, 501, 503, 503, 496, 526, 507, 508, 533, 496, 506, 519, 499, 528, 520, 516, 511, 517, 507, 530, 495, 491, 484, 520, 544, 528, 520, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[504, 543, 509, 512, 530, 543, 508, 492, 533, 525, 535, 514, 523, 498, 512, 524, 481, 523, 506, 499, 530, 518, 515, 518, 478, 531, 521, 515, 525, 520, 518, 530, 530, 499, 491, 513, 481, 541, 531, 539, 502, 515, 494, 514, 489, 497, 501, 522, 511, 468, 519, 487, 507, 524, 527, 502, 509, 507, 509, 502, 507, 516, 497, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[514, 514, 537, 515, 520, 499, 482, 523, 554, 531, 479, 488, 487, 504, 542, 520, 536, 505, 507, 512, 550, 493, 512, 517, 506, 493, 479, 509, 525, 492, 526, 502, 516, 508, 523, 460, 529, 568, 561, 495, 526, 508, 509, 488, 533, 497, 521, 475, 483, 519, 512, 518, 522, 522, 495, 519, 479, 506, 541, 539, 499, 504, 491, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[544, 466, 502, 530, 502, 491, 517, 521, 508, 560, 530, 494, 551, 536, 496, 531, 490, 505, 489, 522, 496, 503, 534, 531, 492, 526, 501, 539, 504, 526, 536, 512, 511, 513, 483, 488, 563, 497, 489, 513, 491, 512, 550, 519, 509, 510, 542, 523, 528, 480, 494, 496, 508, 505, 516, 497, 493, 482, 514, 542, 475, 518, 532, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[534, 489, 501, 521, 497, 505, 498, 533, 506, 484, 533, 551, 552, 537, 519, 506, 507, 510, 496, 478, 537, 545, 541, 515, 508, 530, 551, 547, 487, 537, 498, 498, 538, 505, 512, 476, 500, 516, 496, 520, 465, 504, 504, 531, 514, 494, 541, 508, 508, 495, 492, 529, 534, 460, 500, 515, 536, 499, 517, 470, 509, 527, 480, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[510, 503, 529, 483, 520, 522, 492, 483, 514, 519, 546, 472, 518, 523, 491, 497, 481, 519, 467, 477, 485, 503, 521, 519, 498, 493, 476, 507, 547, 472, 525, 525, 541, 492, 551, 511, 524, 536, 520, 528, 535, 514, 503, 493, 547, 532, 491, 515, 531, 523, 481, 503, 500, 497, 531, 568, 538, 512, 543, 494, 514, 503, 529, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[495, 541, 476, 507, 486, 502, 535, 530, 516, 525, 498, 511, 531, 520, 513, 538, 488, 514, 520, 505, 572, 486, 526, 540, 515, 497, 522, 500, 488, 485, 538, 512, 523, 507, 496, 492, 504, 518, 523, 525, 518, 476, 528, 540, 484, 521, 484, 490, 491, 513, 540, 527, 525, 542, 510, 492, 493, 518, 523, 511, 475, 523, 508, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 500, 495, 501, 535, 509, 516, 512, 20, 0, 0, 0, 0, 0, 0, 0, 494, 495, 511, 525, 495, 494, 527, 461, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 547, 511, 512, 511, 513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 492, 481, 518, 505, 554, 534, 0, 0, 0, 0, 0, 0, 0, 0, 502, 497, 520, 469, 512, 506, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0, 517, 508, 511, 523, 539, 493, 536, 513, 6, 0, 0, 0, 0, 0, 0, 0, 538, 514, 500, 544, 502, 520, 495, 471, 7, 3, 0, 0, 0, 0, 0, 0, 558, 453, 506, 534, 527, 500, 486, 505, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4002, 4189, 4103, 4067, 4146, 4094, 4085] avg=4096.0000 max=4189 max_rank=2 min=4002 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 543, 509, 512, 530, 543, 508, 492, 21, 0, 0, 0, 0, 0, 0, 0, 501, 525, 535, 514, 523, 498, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 506, 499, 518, 518, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 478, 531, 521, 515, 525, 520, 518, 530, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 491, 513, 481, 541, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 494, 514, 489, 497, 501, 522, 7, 0, 0, 0, 0, 0, 0, 0, 511, 468, 519, 487, 507, 524, 527, 502, 7, 3, 0, 0, 0, 0, 0, 0, 509, 507, 509, 502, 507, 516, 497, 484, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4132, 4078, 4138, 4115, 4041, 4055, 4047] avg=4096.0000 max=4162 max_rank=0 min=4041 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[503, 497, 514, 528, 507, 505, 476, 501, 20, 0, 0, 0, 0, 0, 0, 0, 488, 509, 527, 493, 471, 490, 537, 548, 0, 0, 0, 0, 0, 0, 0, 0, 539, 480, 531, 520, 512, 489, 537, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 514, 510, 523, 548, 491, 499, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 490, 543, 489, 504, 501, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 507, 508, 533, 496, 506, 519, 7, 0, 0, 0, 0, 0, 0, 0, 499, 528, 520, 516, 511, 517, 507, 530, 7, 3, 0, 0, 0, 0, 0, 0, 495, 491, 484, 520, 544, 528, 520, 531, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4063, 4115, 4105, 4069, 4098, 4138, 4129] avg=4096.0000 max=4138 max_rank=6 min=4051 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 514, 537, 515, 520, 499, 482, 523, 21, 0, 0, 0, 0, 0, 0, 0, 522, 531, 479, 488, 487, 504, 542, 520, 0, 0, 0, 0, 0, 0, 0, 0, 536, 505, 507, 512, 537, 493, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 479, 509, 525, 492, 526, 502, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 523, 460, 529, 568, 561, 495, 0, 0, 0, 0, 0, 0, 0, 0, 526, 508, 509, 488, 533, 497, 521, 475, 6, 0, 0, 0, 0, 0, 0, 0, 483, 519, 512, 518, 522, 522, 495, 519, 8, 3, 0, 0, 0, 0, 0, 0, 479, 506, 541, 539, 499, 504, 491, 529, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4073, 4119, 4032, 4151, 4063, 4101, 4104] avg=4096.0000 max=4151 max_rank=4 min=4032 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 489, 501, 521, 497, 505, 498, 533, 19, 0, 0, 0, 0, 0, 0, 0, 477, 484, 533, 551, 552, 537, 519, 506, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 496, 478, 524, 545, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 508, 530, 551, 547, 487, 537, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 512, 476, 500, 516, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 465, 504, 504, 531, 514, 494, 541, 508, 7, 0, 0, 0, 0, 0, 0, 0, 508, 495, 492, 529, 534, 460, 500, 515, 8, 3, 0, 0, 0, 0, 0, 0, 536, 499, 517, 470, 509, 527, 480, 522, 10, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4159, 4116, 4156, 4053, 4068, 4044, 4075] avg=4096.0000 max=4159 max_rank=1 min=4044 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[544, 466, 502, 530, 502, 491, 517, 521, 20, 0, 0, 0, 0, 0, 0, 0, 478, 560, 530, 494, 551, 536, 496, 531, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 489, 522, 485, 503, 534, 531, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 501, 539, 504, 526, 536, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 483, 488, 563, 497, 489, 513, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 550, 519, 509, 510, 542, 523, 6, 0, 0, 0, 0, 0, 0, 0, 528, 480, 494, 496, 508, 505, 516, 497, 7, 3, 0, 0, 0, 0, 0, 0, 493, 482, 514, 542, 475, 518, 532, 490, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4176, 4059, 4136, 4048, 4162, 4034, 4060] avg=4096.0000 max=4176 max_rank=1 min=4034 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 541, 476, 507, 486, 502, 535, 530, 24, 0, 0, 0, 0, 0, 0, 0, 477, 525, 498, 511, 531, 520, 513, 538, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 520, 505, 552, 486, 526, 540, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 522, 500, 488, 485, 536, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 496, 492, 504, 518, 523, 525, 0, 0, 0, 0, 0, 0, 0, 0, 518, 476, 528, 540, 484, 521, 484, 490, 10, 2, 0, 0, 0, 0, 0, 0, 491, 513, 540, 527, 525, 542, 510, 492, 12, 6, 0, 0, 0, 0, 0, 0, 493, 518, 523, 511, 475, 523, 508, 516, 15, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4113, 4131, 4055, 4072, 4053, 4158, 4090] avg=4096.0000 max=4158 max_rank=6 min=4053 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[510, 503, 529, 483, 520, 522, 492, 483, 20, 0, 0, 0, 0, 0, 0, 0, 484, 519, 546, 472, 518, 523, 491, 497, 0, 0, 0, 0, 0, 0, 0, 0, 481, 519, 467, 477, 474, 503, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 498, 493, 476, 507, 547, 472, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 531, 492, 551, 511, 524, 536, 520, 528, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 503, 493, 547, 532, 491, 515, 7, 0, 0, 0, 0, 0, 0, 0, 531, 523, 481, 503, 500, 497, 531, 568, 7, 3, 0, 0, 0, 0, 0, 0, 538, 512, 543, 494, 514, 503, 529, 531, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4050, 3961, 4043, 4193, 4137, 4144, 4178] avg=4096.0000 max=4193 max_rank=4 min=3961 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[488, 519, 517, 532, 521, 508, 500, 543, 518, 507, 544, 498, 518, 546, 542, 550, 545, 519, 503, 502, 492, 506, 502, 507, 475, 500, 499, 506, 507, 563, 546, 509, 502, 549, 550, 535, 533, 492, 455, 516, 482, 537, 529, 537, 536, 534, 512, 499, 526, 481, 487, 526, 488, 481, 494, 503, 503, 483, 506, 511, 495, 485, 481, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[545, 503, 488, 523, 553, 526, 488, 536, 512, 507, 546, 503, 507, 511, 524, 467, 502, 513, 477, 520, 555, 486, 506, 512, 546, 465, 537, 481, 522, 551, 522, 528, 501, 509, 505, 515, 534, 526, 552, 502, 537, 513, 501, 509, 529, 517, 503, 471, 519, 480, 504, 516, 495, 492, 511, 502, 515, 530, 509, 491, 496, 529, 479, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[489, 492, 494, 563, 500, 466, 537, 531, 514, 538, 498, 533, 515, 468, 495, 516, 521, 517, 543, 503, 482, 525, 494, 530, 522, 503, 491, 493, 499, 520, 536, 487, 462, 501, 552, 517, 505, 534, 521, 496, 497, 526, 497, 547, 492, 515, 528, 521, 511, 510, 505, 517, 504, 525, 479, 529, 533, 529, 509, 533, 524, 502, 513, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[541, 567, 494, 544, 510, 497, 487, 510, 515, 549, 529, 558, 513, 501, 487, 500, 496, 524, 499, 493, 542, 488, 556, 474, 501, 525, 529, 516, 485, 505, 491, 521, 512, 499, 514, 490, 550, 518, 524, 466, 514, 512, 512, 516, 527, 511, 511, 523, 524, 508, 512, 514, 484, 467, 517, 509, 467, 506, 508, 556, 486, 523, 524, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[482, 541, 504, 470, 479, 508, 524, 499, 484, 499, 506, 503, 488, 541, 499, 495, 528, 513, 498, 506, 514, 515, 545, 495, 530, 499, 512, 505, 501, 513, 515, 497, 478, 486, 511, 528, 530, 491, 510, 524, 506, 508, 509, 477, 545, 517, 551, 515, 525, 511, 499, 490, 514, 517, 512, 527, 554, 527, 510, 552, 517, 560, 522, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[503, 482, 516, 536, 524, 566, 507, 498, 514, 542, 530, 501, 505, 522, 478, 488, 499, 480, 539, 505, 507, 504, 567, 510, 503, 521, 509, 496, 506, 533, 522, 494, 558, 540, 492, 546, 488, 508, 484, 490, 528, 521, 498, 531, 471, 515, 485, 507, 534, 538, 490, 498, 515, 542, 496, 516, 507, 530, 519, 445, 552, 504, 489, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[545, 522, 497, 540, 477, 531, 489, 512, 513, 519, 494, 504, 507, 489, 541, 519, 524, 494, 506, 517, 529, 487, 509, 545, 524, 506, 541, 538, 496, 498, 485, 515, 500, 462, 483, 547, 532, 528, 475, 541, 536, 495, 528, 533, 522, 478, 486, 475, 485, 494, 539, 514, 522, 485, 531, 515, 514, 536, 511, 521, 555, 478, 528, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[504, 532, 534, 515, 488, 541, 519, 546, 505, 520, 497, 523, 514, 526, 544, 515, 513, 529, 479, 522, 506, 514, 451, 520, 518, 484, 484, 487, 519, 514, 492, 562, 508, 527, 491, 524, 496, 518, 523, 528, 498, 505, 498, 492, 546, 537, 495, 542, 517, 514, 506, 527, 482, 524, 500, 506, 512, 498, 487, 495, 493, 516, 511, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 503, 488, 502, 553, 526, 488, 536, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 546, 503, 507, 511, 524, 467, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 477, 520, 555, 486, 506, 512, 5, 1, 0, 0, 0, 0, 0, 0, 546, 465, 537, 481, 522, 551, 522, 528, 8, 0, 0, 0, 0, 0, 0, 0, 501, 509, 505, 514, 534, 526, 552, 502, 0, 0, 0, 0, 0, 0, 0, 0, 537, 513, 501, 509, 521, 517, 503, 471, 0, 0, 0, 0, 0, 0, 0, 0, 519, 480, 504, 516, 495, 492, 511, 502, 21, 10, 0, 0, 0, 0, 0, 0, 515, 530, 509, 491, 496, 529, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4067, 4077, 4160, 4143, 4072, 4050, 4058] avg=4096.0000 max=4160 max_rank=3 min=4050 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[488, 519, 517, 510, 521, 508, 500, 543, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 544, 498, 518, 546, 542, 550, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 503, 502, 492, 506, 502, 507, 4, 2, 0, 0, 0, 0, 0, 0, 475, 500, 499, 506, 507, 563, 546, 509, 8, 0, 0, 0, 0, 0, 0, 0, 502, 549, 550, 533, 533, 492, 455, 516, 0, 0, 0, 0, 0, 0, 0, 0, 482, 537, 529, 537, 528, 534, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 526, 481, 487, 526, 488, 481, 494, 503, 22, 10, 0, 0, 0, 0, 0, 0, 503, 483, 506, 511, 495, 485, 481, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4213, 4082, 4113, 4130, 4158, 4018, 3948] avg=4096.0000 max=4213 max_rank=1 min=3948 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[489, 492, 494, 540, 500, 466, 537, 531, 0, 0, 0, 0, 0, 0, 0, 0, 514, 527, 498, 533, 515, 468, 495, 516, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 543, 503, 482, 525, 494, 530, 5, 1, 0, 0, 0, 0, 0, 0, 522, 503, 491, 493, 499, 520, 536, 487, 8, 0, 0, 0, 0, 0, 0, 0, 462, 501, 552, 516, 505, 534, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 497, 547, 484, 515, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 505, 517, 504, 525, 479, 529, 23, 11, 0, 0, 0, 0, 0, 0, 533, 529, 509, 533, 524, 502, 513, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4066, 4121, 4059, 4087, 4115, 4114, 4157] avg=4096.0000 max=4157 max_rank=7 min=4049 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[541, 567, 494, 522, 510, 497, 487, 510, 0, 0, 0, 0, 0, 0, 0, 0, 515, 538, 529, 558, 513, 501, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 499, 493, 542, 488, 556, 474, 5, 1, 0, 0, 0, 0, 0, 0, 501, 525, 529, 516, 485, 505, 491, 521, 8, 0, 0, 0, 0, 0, 0, 0, 512, 499, 514, 489, 550, 518, 524, 466, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 512, 516, 519, 511, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 512, 514, 484, 467, 517, 509, 22, 11, 0, 0, 0, 0, 0, 0, 467, 506, 508, 556, 486, 523, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4141, 4078, 4081, 4072, 4118, 4068, 4082] avg=4096.0000 max=4141 max_rank=1 min=4068 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[482, 541, 504, 448, 479, 508, 524, 499, 0, 0, 0, 0, 0, 0, 0, 0, 484, 486, 506, 503, 488, 541, 499, 495, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 498, 506, 514, 515, 545, 495, 7, 6, 7, 0, 0, 0, 0, 0, 530, 499, 512, 505, 501, 513, 515, 497, 13, 0, 0, 0, 0, 0, 0, 0, 478, 486, 511, 522, 530, 491, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 509, 477, 523, 517, 551, 515, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 499, 490, 514, 517, 512, 527, 22, 13, 2, 0, 0, 0, 0, 0, 554, 527, 510, 552, 517, 560, 522, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4002, 4134, 4085, 4052, 4106, 4132, 4272] avg=4096.0000 max=4272 max_rank=7 min=3985 min_rank=0 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[503, 482, 516, 514, 524, 566, 507, 498, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 530, 501, 505, 522, 478, 488, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 539, 505, 507, 504, 567, 510, 5, 2, 0, 0, 0, 0, 0, 0, 503, 521, 509, 496, 506, 533, 522, 494, 7, 0, 0, 0, 0, 0, 0, 0, 558, 540, 492, 544, 488, 508, 484, 490, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 498, 531, 464, 515, 485, 507, 0, 0, 0, 0, 0, 0, 0, 0, 534, 538, 490, 498, 515, 542, 496, 516, 22, 11, 0, 0, 0, 0, 0, 0, 507, 530, 519, 445, 552, 504, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4069, 4118, 4091, 4104, 4049, 4162, 4065] avg=4096.0000 max=4162 max_rank=6 min=4049 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 522, 497, 518, 477, 531, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 494, 504, 507, 489, 541, 519, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 506, 517, 529, 487, 509, 545, 5, 2, 0, 0, 0, 0, 0, 0, 524, 506, 541, 538, 496, 498, 485, 515, 8, 0, 0, 0, 0, 0, 0, 0, 500, 462, 483, 545, 532, 528, 475, 541, 0, 0, 0, 0, 0, 0, 0, 0, 536, 495, 528, 533, 514, 478, 486, 475, 0, 0, 0, 0, 0, 0, 0, 0, 485, 494, 539, 514, 522, 485, 531, 515, 22, 10, 0, 0, 0, 0, 0, 0, 514, 536, 511, 521, 555, 478, 528, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4076, 4118, 4111, 4066, 4045, 4117, 4144] avg=4096.0000 max=4144 max_rank=7 min=4045 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 532, 534, 494, 488, 541, 519, 546, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 497, 523, 514, 526, 544, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 479, 522, 506, 514, 451, 520, 5, 1, 0, 0, 0, 0, 0, 0, 518, 484, 484, 487, 519, 514, 492, 562, 8, 0, 0, 0, 0, 0, 0, 0, 508, 527, 491, 523, 496, 518, 523, 528, 0, 0, 0, 0, 0, 0, 0, 0, 498, 505, 498, 492, 538, 537, 495, 542, 0, 0, 0, 0, 0, 0, 0, 0, 517, 514, 506, 527, 482, 524, 500, 506, 21, 10, 0, 0, 0, 0, 0, 0, 512, 498, 487, 495, 493, 516, 511, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4134, 4040, 4068, 4114, 4105, 4107, 4042] avg=4096.0000 max=4158 max_rank=0 min=4040 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[480, 525, 514, 505, 537, 515, 479, 517, 526, 479, 523, 554, 498, 504, 471, 499, 528, 538, 522, 535, 473, 489, 477, 512, 514, 529, 514, 539, 524, 505, 507, 543, 509, 488, 562, 497, 518, 502, 514, 563, 509, 550, 510, 522, 521, 494, 490, 490, 501, 511, 526, 475, 552, 514, 494, 490, 523, 493, 505, 553, 515, 505, 508, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[508, 483, 501, 496, 502, 521, 530, 516, 526, 532, 510, 491, 493, 513, 548, 480, 510, 504, 513, 539, 502, 519, 520, 510, 533, 496, 489, 476, 509, 481, 514, 487, 517, 503, 517, 498, 512, 559, 498, 495, 499, 484, 527, 502, 498, 489, 513, 557, 500, 523, 558, 545, 543, 520, 513, 525, 538, 495, 509, 524, 509, 518, 505, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[530, 533, 479, 518, 557, 506, 508, 516, 498, 520, 536, 531, 515, 469, 511, 517, 523, 483, 509, 495, 500, 506, 538, 533, 480, 508, 464, 507, 487, 525, 500, 512, 480, 516, 493, 562, 490, 523, 525, 527, 519, 544, 495, 544, 483, 522, 539, 565, 472, 473, 521, 523, 461, 522, 468, 521, 530, 522, 515, 510, 510, 527, 503, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[511, 546, 526, 513, 524, 519, 508, 516, 500, 517, 490, 516, 511, 508, 521, 552, 505, 512, 504, 527, 518, 507, 492, 491, 525, 513, 507, 535, 505, 531, 556, 560, 483, 515, 520, 475, 517, 484, 458, 515, 531, 519, 486, 500, 520, 511, 518, 489, 525, 469, 552, 530, 503, 512, 487, 500, 494, 492, 561, 530, 482, 499, 529, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[567, 509, 495, 476, 520, 511, 508, 476, 532, 502, 485, 505, 548, 524, 514, 539, 489, 502, 537, 516, 492, 555, 497, 506, 481, 493, 516, 534, 510, 494, 514, 483, 500, 561, 527, 516, 513, 525, 512, 491, 511, 503, 540, 504, 529, 507, 530, 516, 487, 525, 517, 510, 511, 543, 486, 485, 526, 518, 469, 533, 483, 504, 532, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[526, 495, 536, 510, 523, 524, 513, 496, 515, 492, 510, 527, 494, 535, 532, 494, 541, 494, 493, 511, 504, 535, 548, 499, 485, 493, 502, 509, 521, 513, 508, 508, 495, 546, 563, 543, 508, 456, 503, 502, 492, 530, 518, 503, 536, 493, 535, 527, 531, 496, 496, 516, 510, 484, 509, 525, 512, 498, 507, 525, 533, 499, 493, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[514, 510, 558, 485, 561, 473, 464, 504, 501, 505, 485, 548, 477, 533, 521, 515, 517, 529, 481, 536, 524, 477, 500, 555, 487, 531, 513, 511, 493, 535, 470, 505, 513, 526, 514, 535, 499, 531, 541, 502, 527, 548, 494, 511, 487, 480, 504, 548, 504, 528, 546, 531, 524, 497, 488, 524, 483, 531, 537, 508, 524, 491, 473, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[479, 490, 487, 496, 515, 509, 530, 511, 538, 489, 532, 530, 524, 476, 558, 507, 538, 508, 508, 508, 507, 513, 509, 524, 486, 517, 534, 544, 504, 493, 520, 538, 540, 527, 511, 515, 526, 503, 544, 506, 523, 511, 515, 497, 512, 531, 493, 493, 495, 495, 502, 473, 488, 513, 517, 509, 489, 535, 537, 505, 489, 528, 512, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 533, 479, 518, 557, 506, 508, 516, 4, 2, 0, 0, 0, 0, 0, 0, 498, 520, 536, 522, 515, 469, 511, 517, 0, 0, 0, 0, 0, 0, 0, 0, 523, 483, 509, 493, 500, 506, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0, 480, 508, 464, 507, 487, 525, 500, 512, 13, 5, 0, 0, 0, 0, 0, 0, 480, 516, 475, 562, 490, 523, 525, 527, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 495, 544, 483, 522, 539, 565, 0, 0, 0, 0, 0, 0, 0, 0, 472, 473, 521, 523, 461, 522, 468, 521, 18, 1, 0, 0, 0, 0, 0, 0, 530, 522, 515, 510, 510, 527, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4088, 4085, 4001, 4098, 4197, 3980, 4166] avg=4096.0000 max=4197 max_rank=5 min=3980 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 483, 501, 496, 502, 521, 530, 516, 4, 2, 0, 0, 0, 0, 0, 0, 526, 532, 510, 483, 493, 513, 548, 480, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 513, 537, 502, 519, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 533, 496, 489, 476, 509, 481, 514, 487, 12, 4, 0, 0, 0, 0, 0, 0, 517, 503, 498, 498, 512, 559, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 499, 471, 527, 502, 498, 489, 513, 557, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 558, 545, 543, 520, 513, 525, 19, 1, 0, 0, 0, 0, 0, 0, 538, 495, 509, 524, 509, 518, 505, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4085, 4115, 4001, 4080, 4056, 4247, 4121] avg=4096.0000 max=4247 max_rank=6 min=4001 min_rank=3 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[480, 525, 514, 505, 537, 515, 479, 517, 4, 2, 0, 0, 0, 0, 0, 0, 526, 479, 523, 545, 498, 504, 471, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 538, 522, 533, 473, 489, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 514, 539, 524, 505, 507, 543, 14, 5, 0, 0, 0, 0, 0, 0, 509, 488, 541, 497, 518, 502, 514, 563, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 510, 522, 521, 494, 490, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 511, 526, 475, 552, 514, 494, 490, 21, 1, 0, 0, 0, 0, 0, 0, 523, 493, 505, 553, 515, 505, 508, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4045, 4072, 4194, 4132, 4071, 4085, 4091] avg=4096.0000 max=4194 max_rank=3 min=4045 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 546, 526, 513, 524, 519, 508, 516, 4, 2, 0, 0, 0, 0, 0, 0, 500, 517, 490, 507, 511, 508, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 505, 512, 504, 525, 518, 507, 492, 491, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 507, 535, 505, 531, 556, 560, 13, 5, 0, 0, 0, 0, 0, 0, 483, 515, 501, 475, 517, 484, 458, 515, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 486, 500, 520, 511, 518, 489, 0, 0, 0, 0, 0, 0, 0, 0, 525, 469, 552, 530, 503, 512, 487, 500, 19, 1, 0, 0, 0, 0, 0, 0, 494, 492, 561, 530, 482, 499, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4106, 4054, 4250, 3948, 4060, 4098, 4083] avg=4096.0000 max=4250 max_rank=3 min=3948 min_rank=4 imbalance_ratio=1.037598 max_load_reduction=28518 max_load_reduction_pct=0.870300 moved_to_echo=16189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[526, 495, 536, 510, 523, 524, 513, 496, 4, 2, 0, 0, 0, 0, 0, 0, 515, 492, 510, 518, 494, 535, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0, 541, 494, 493, 509, 504, 535, 548, 499, 0, 0, 0, 0, 0, 0, 0, 0, 485, 493, 502, 509, 521, 513, 508, 508, 13, 5, 0, 0, 0, 0, 0, 0, 495, 546, 542, 543, 508, 456, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 518, 503, 536, 493, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 496, 516, 510, 484, 509, 525, 21, 1, 0, 0, 0, 0, 0, 0, 512, 498, 507, 525, 533, 499, 493, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4090, 4123, 4057, 4095, 4120, 4089, 4065] avg=4096.0000 max=4129 max_rank=0 min=4057 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[567, 509, 495, 476, 520, 511, 508, 476, 9, 5, 2, 0, 0, 0, 0, 0, 532, 502, 485, 489, 548, 524, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 502, 537, 511, 492, 555, 497, 506, 0, 0, 0, 0, 0, 0, 0, 0, 481, 493, 516, 534, 510, 494, 514, 483, 16, 7, 0, 0, 0, 0, 0, 0, 500, 561, 502, 516, 513, 525, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 540, 504, 529, 507, 530, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 525, 517, 510, 511, 543, 486, 485, 25, 6, 0, 0, 0, 0, 0, 0, 526, 518, 469, 531, 483, 504, 532, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4133, 4089, 4048, 4120, 4118, 4095, 4087] avg=4096.0000 max=4133 max_rank=1 min=4048 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[479, 490, 487, 496, 515, 509, 530, 511, 4, 2, 0, 0, 0, 0, 0, 0, 538, 489, 532, 521, 524, 476, 558, 507, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 508, 506, 507, 513, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 486, 517, 534, 544, 504, 493, 520, 538, 13, 5, 0, 0, 0, 0, 0, 0, 540, 527, 492, 515, 526, 503, 544, 506, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 515, 497, 512, 531, 493, 493, 0, 0, 0, 0, 0, 0, 0, 0, 495, 495, 502, 473, 488, 513, 517, 509, 19, 1, 0, 0, 0, 0, 0, 0, 489, 535, 537, 505, 489, 528, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4145, 4113, 4154, 4153, 4061, 4012, 4107] avg=4096.0000 max=4154 max_rank=3 min=4012 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 510, 558, 485, 561, 473, 464, 504, 4, 2, 0, 0, 0, 0, 0, 0, 501, 505, 485, 539, 477, 533, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 529, 481, 534, 524, 477, 500, 555, 0, 0, 0, 0, 0, 0, 0, 0, 487, 531, 513, 511, 493, 535, 470, 505, 13, 5, 0, 0, 0, 0, 0, 0, 513, 526, 495, 535, 499, 531, 541, 502, 0, 0, 0, 0, 0, 0, 0, 0, 527, 534, 494, 511, 487, 480, 504, 548, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 546, 531, 524, 497, 488, 524, 19, 1, 0, 0, 0, 0, 0, 0, 483, 531, 537, 508, 524, 491, 473, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4076, 4117, 4063, 4142, 4085, 4162, 4048] avg=4096.0000 max=4162 max_rank=6 min=4048 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[467, 506, 520, 518, 519, 505, 515, 557, 499, 510, 515, 472, 506, 499, 546, 486, 521, 513, 507, 547, 515, 478, 517, 498, 523, 496, 513, 487, 508, 529, 499, 486, 558, 506, 469, 557, 512, 477, 520, 548, 486, 517, 510, 518, 480, 520, 521, 495, 542, 514, 532, 463, 549, 524, 523, 522, 556, 499, 509, 527, 508, 492, 518, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[508, 495, 525, 534, 519, 507, 533, 492, 521, 518, 547, 570, 538, 477, 504, 484, 510, 503, 526, 494, 539, 478, 542, 465, 517, 489, 508, 490, 477, 538, 487, 469, 487, 488, 545, 524, 483, 538, 508, 501, 524, 509, 500, 540, 495, 518, 536, 499, 508, 542, 514, 509, 517, 525, 522, 515, 526, 500, 517, 502, 518, 533, 517, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[517, 540, 509, 495, 499, 517, 493, 504, 533, 514, 530, 525, 500, 538, 562, 545, 535, 502, 484, 502, 481, 521, 515, 540, 516, 515, 518, 514, 495, 507, 511, 500, 516, 500, 514, 471, 495, 505, 547, 528, 489, 512, 523, 519, 534, 504, 502, 562, 559, 459, 456, 484, 500, 537, 462, 503, 515, 510, 519, 530, 513, 522, 494, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[490, 517, 525, 493, 535, 523, 503, 503, 497, 566, 513, 477, 506, 537, 500, 511, 497, 494, 496, 505, 513, 510, 517, 502, 492, 500, 526, 486, 510, 475, 516, 489, 528, 492, 529, 528, 523, 563, 515, 484, 500, 545, 514, 498, 516, 505, 522, 511, 493, 585, 573, 464, 536, 518, 491, 515, 509, 513, 501, 510, 497, 534, 508, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[488, 481, 492, 472, 499, 529, 516, 496, 509, 543, 507, 535, 506, 482, 545, 523, 503, 504, 518, 506, 523, 540, 508, 535, 529, 550, 457, 508, 562, 491, 489, 484, 528, 507, 501, 528, 551, 521, 507, 569, 494, 500, 482, 509, 510, 484, 536, 490, 531, 487, 507, 535, 535, 546, 524, 493, 510, 508, 491, 524, 498, 496, 521, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[495, 535, 499, 505, 506, 532, 497, 515, 505, 526, 481, 501, 515, 478, 510, 506, 509, 489, 526, 496, 499, 552, 516, 547, 522, 465, 521, 510, 501, 487, 488, 549, 529, 530, 489, 536, 485, 532, 536, 492, 509, 508, 513, 491, 512, 527, 552, 505, 504, 473, 557, 520, 501, 514, 499, 507, 502, 504, 558, 506, 539, 525, 494, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[493, 516, 488, 514, 512, 512, 527, 535, 539, 500, 533, 465, 542, 499, 503, 524, 528, 495, 523, 537, 497, 555, 494, 523, 473, 522, 518, 498, 474, 507, 506, 496, 529, 493, 507, 531, 559, 517, 513, 516, 564, 490, 517, 537, 529, 498, 493, 516, 473, 482, 492, 504, 543, 519, 483, 505, 512, 523, 517, 483, 513, 521, 542, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[467, 506, 520, 518, 519, 505, 515, 557, 7, 4, 2, 2, 0, 0, 0, 0, 499, 510, 515, 472, 506, 499, 527, 486, 0, 0, 0, 0, 0, 0, 0, 0, 521, 513, 507, 547, 515, 474, 517, 498, 0, 0, 0, 0, 0, 0, 0, 0, 523, 496, 513, 487, 508, 529, 499, 486, 36, 19, 6, 0, 0, 0, 0, 0, 522, 506, 469, 557, 512, 477, 520, 548, 0, 0, 0, 0, 0, 0, 0, 0, 486, 517, 510, 518, 480, 520, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 542, 514, 532, 463, 547, 524, 523, 522, 0, 0, 0, 0, 0, 0, 0, 0, 549, 499, 509, 527, 508, 492, 518, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4014, 4092, 4102, 4111, 4039, 4167, 4121] avg=4096.0000 max=4167 max_rank=6 min=4014 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 540, 509, 495, 499, 517, 493, 504, 6, 5, 2, 2, 0, 0, 0, 0, 533, 514, 530, 525, 500, 538, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 484, 502, 481, 516, 515, 540, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 518, 514, 495, 507, 511, 500, 33, 20, 6, 0, 0, 0, 0, 0, 483, 500, 514, 471, 495, 505, 547, 528, 0, 0, 0, 0, 0, 0, 0, 0, 489, 512, 523, 519, 534, 504, 494, 562, 0, 0, 0, 0, 0, 0, 0, 0, 559, 459, 456, 484, 498, 537, 462, 503, 0, 0, 0, 0, 0, 0, 0, 0, 509, 510, 519, 530, 513, 522, 494, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4227, 4075, 4135, 4043, 4137, 3958, 4104] avg=4096.0000 max=4227 max_rank=1 min=3958 min_rank=6 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 495, 525, 534, 519, 507, 533, 492, 6, 4, 2, 2, 0, 0, 0, 0, 521, 518, 547, 570, 538, 477, 486, 484, 0, 0, 0, 0, 0, 0, 0, 0, 510, 503, 526, 494, 539, 474, 542, 465, 0, 0, 0, 0, 0, 0, 0, 0, 517, 489, 508, 490, 477, 538, 487, 469, 31, 18, 6, 0, 0, 0, 0, 0, 456, 488, 545, 524, 483, 538, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0, 524, 509, 500, 540, 495, 518, 528, 499, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 514, 509, 515, 525, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 520, 500, 517, 502, 518, 533, 517, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4141, 4053, 4030, 4043, 4113, 4150, 4111] avg=4096.0000 max=4150 max_rank=6 min=4030 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[495, 535, 499, 505, 506, 532, 497, 515, 6, 5, 2, 2, 0, 0, 0, 0, 505, 526, 481, 501, 515, 478, 492, 506, 0, 0, 0, 0, 0, 0, 0, 0, 509, 489, 526, 496, 499, 547, 516, 547, 0, 0, 0, 0, 0, 0, 0, 0, 522, 465, 521, 510, 501, 487, 488, 549, 34, 18, 6, 0, 0, 0, 0, 0, 495, 530, 489, 536, 485, 532, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 513, 491, 512, 527, 544, 505, 0, 0, 0, 0, 0, 0, 0, 0, 504, 473, 557, 520, 499, 514, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 558, 506, 539, 525, 494, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4004, 4129, 4101, 4095, 4109, 4073, 4158] avg=4096.0000 max=4158 max_rank=7 min=4004 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[490, 517, 525, 493, 535, 523, 503, 503, 6, 5, 2, 2, 0, 0, 0, 0, 497, 566, 513, 477, 506, 537, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 496, 505, 513, 505, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 526, 486, 510, 475, 516, 489, 34, 18, 6, 0, 0, 0, 0, 0, 494, 492, 529, 528, 523, 563, 515, 484, 0, 0, 0, 0, 0, 0, 0, 0, 500, 545, 514, 498, 516, 505, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 493, 585, 573, 464, 534, 518, 491, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 501, 510, 497, 534, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4089, 4029, 4052, 4128, 4103, 4173, 4090] avg=4096.0000 max=4173 max_rank=6 min=4029 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[488, 481, 492, 472, 499, 529, 516, 496, 11, 9, 8, 5, 0, 0, 0, 0, 509, 543, 507, 535, 506, 482, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 518, 506, 523, 531, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 529, 550, 457, 508, 562, 491, 489, 484, 37, 22, 11, 0, 0, 0, 0, 0, 491, 507, 501, 528, 551, 521, 507, 569, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 482, 509, 510, 484, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 531, 487, 507, 535, 530, 546, 524, 493, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 491, 524, 498, 496, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4128, 4128, 4140, 4175, 3986, 4153, 4052] avg=4096.0000 max=4175 max_rank=4 min=3986 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 516, 488, 514, 512, 512, 527, 535, 6, 5, 2, 2, 0, 0, 0, 0, 539, 500, 533, 465, 542, 499, 485, 524, 0, 0, 0, 0, 0, 0, 0, 0, 528, 495, 523, 537, 497, 550, 494, 523, 0, 0, 0, 0, 0, 0, 0, 0, 473, 522, 518, 498, 474, 507, 506, 496, 34, 18, 6, 0, 0, 0, 0, 0, 495, 493, 507, 531, 559, 517, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 564, 490, 517, 537, 529, 498, 485, 516, 0, 0, 0, 0, 0, 0, 0, 0, 473, 482, 492, 504, 541, 519, 483, 505, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 517, 483, 513, 521, 542, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4087, 4147, 4052, 4131, 4136, 3999, 4104] avg=4096.0000 max=4147 max_rank=2 min=3999 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[518, 522, 521, 526, 487, 501, 518, 501, 518, 485, 478, 547, 501, 506, 540, 522, 525, 551, 473, 513, 490, 522, 540, 506, 512, 517, 509, 510, 470, 536, 516, 527, 538, 521, 474, 510, 563, 504, 494, 472, 525, 525, 505, 508, 508, 503, 530, 549, 509, 486, 520, 511, 512, 489, 531, 539, 514, 471, 524, 523, 519, 490, 498, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[518, 522, 521, 526, 487, 501, 518, 501, 6, 5, 2, 2, 0, 0, 0, 0, 518, 485, 478, 547, 501, 506, 521, 522, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 473, 513, 490, 517, 540, 506, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 509, 510, 470, 536, 516, 527, 34, 19, 6, 0, 0, 0, 0, 0, 504, 521, 474, 510, 563, 504, 494, 472, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 505, 508, 508, 503, 522, 549, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 520, 511, 510, 489, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 524, 523, 519, 490, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4078, 4115, 4156, 4042, 4145, 4095, 4028] avg=4096.0000 max=4156 max_rank=3 min=4028 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[484, 526, 519, 536, 539, 499, 522, 496, 527, 514, 511, 482, 515, 478, 507, 504, 532, 506, 538, 492, 534, 527, 531, 546, 531, 520, 518, 524, 500, 483, 515, 499, 509, 504, 488, 525, 515, 505, 543, 529, 543, 511, 507, 463, 516, 519, 531, 502, 522, 488, 500, 500, 532, 516, 512, 492, 512, 511, 491, 506, 512, 488, 539, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[511, 455, 499, 520, 536, 551, 508, 528, 502, 495, 534, 496, 512, 487, 513, 528, 538, 545, 534, 500, 510, 527, 551, 512, 508, 529, 507, 461, 515, 517, 475, 498, 489, 490, 513, 542, 483, 481, 507, 533, 520, 515, 530, 488, 509, 523, 562, 513, 509, 505, 500, 522, 526, 482, 513, 525, 518, 493, 480, 532, 548, 473, 540, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[503, 530, 509, 465, 493, 513, 509, 534, 549, 530, 547, 501, 493, 510, 503, 482, 527, 529, 535, 472, 503, 508, 501, 555, 480, 504, 528, 508, 537, 498, 503, 510, 525, 479, 537, 509, 469, 500, 482, 548, 490, 513, 502, 492, 516, 516, 523, 535, 515, 501, 515, 503, 529, 530, 495, 499, 522, 526, 508, 562, 512, 519, 504, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[504, 529, 497, 515, 522, 505, 509, 549, 511, 542, 508, 511, 535, 518, 518, 547, 515, 521, 493, 508, 524, 516, 524, 485, 503, 522, 506, 505, 527, 524, 474, 481, 512, 531, 509, 504, 475, 556, 477, 520, 505, 507, 494, 482, 499, 464, 503, 513, 503, 515, 530, 552, 505, 520, 565, 473, 479, 510, 480, 487, 535, 539, 514, 562] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[502, 530, 531, 514, 537, 527, 513, 508, 540, 500, 518, 505, 537, 516, 499, 502, 488, 507, 542, 543, 508, 529, 522, 491, 495, 477, 532, 453, 525, 511, 470, 526, 520, 506, 504, 498, 534, 522, 524, 503, 497, 494, 478, 535, 527, 505, 498, 503, 530, 551, 492, 511, 501, 521, 536, 493, 563, 500, 517, 519, 490, 508, 515, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[517, 484, 507, 528, 525, 536, 537, 502, 503, 478, 521, 492, 514, 516, 513, 510, 513, 544, 530, 507, 529, 514, 516, 479, 501, 508, 484, 513, 506, 508, 515, 521, 539, 561, 500, 478, 508, 520, 505, 510, 523, 498, 555, 537, 523, 535, 507, 492, 489, 527, 494, 507, 509, 492, 504, 507, 537, 518, 510, 500, 495, 503, 499, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[508, 543, 484, 564, 496, 540, 527, 483, 503, 495, 519, 489, 512, 543, 535, 487, 501, 500, 510, 497, 524, 515, 499, 551, 507, 500, 547, 508, 547, 526, 502, 509, 489, 485, 524, 498, 509, 525, 509, 506, 493, 467, 525, 499, 488, 538, 514, 506, 536, 514, 545, 521, 513, 484, 547, 525, 482, 516, 524, 498, 513, 506, 501, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[523, 486, 465, 509, 505, 535, 489, 519, 503, 512, 483, 530, 490, 525, 510, 498, 521, 523, 505, 502, 536, 484, 498, 503, 499, 513, 523, 544, 500, 509, 568, 531, 520, 489, 509, 494, 508, 510, 548, 512, 547, 562, 524, 525, 514, 463, 538, 510, 517, 507, 520, 500, 493, 481, 522, 499, 527, 501, 517, 495, 511, 510, 539, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 526, 519, 536, 539, 486, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 527, 514, 511, 482, 515, 478, 507, 504, 5, 1, 0, 0, 0, 0, 0, 0, 532, 506, 500, 492, 534, 527, 531, 546, 0, 0, 0, 0, 0, 0, 0, 0, 531, 520, 518, 524, 500, 483, 515, 499, 23, 0, 0, 0, 0, 0, 0, 0, 509, 504, 488, 525, 515, 505, 543, 529, 14, 0, 0, 0, 0, 0, 0, 0, 543, 511, 507, 463, 516, 519, 531, 502, 3, 1, 0, 0, 0, 0, 0, 0, 522, 488, 500, 500, 532, 516, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 491, 506, 512, 488, 539, 482, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4044, 4168, 4113, 4132, 4096, 4061, 4046] avg=4096.0000 max=4168 max_rank=2 min=4044 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 530, 509, 465, 493, 499, 509, 534, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 547, 501, 493, 510, 503, 482, 6, 1, 0, 0, 0, 0, 0, 0, 527, 529, 498, 472, 503, 508, 501, 555, 0, 0, 0, 0, 0, 0, 0, 0, 480, 504, 528, 508, 537, 498, 503, 510, 22, 0, 0, 0, 0, 0, 0, 0, 525, 479, 537, 509, 469, 500, 482, 548, 14, 0, 0, 0, 0, 0, 0, 0, 490, 513, 502, 492, 516, 516, 523, 535, 3, 1, 0, 0, 0, 0, 0, 0, 515, 501, 515, 503, 529, 530, 494, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 526, 508, 562, 512, 519, 504, 523, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4122, 4093, 4090, 4063, 4091, 4086, 4181] avg=4096.0000 max=4181 max_rank=7 min=4042 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 455, 499, 520, 536, 537, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 534, 496, 512, 487, 513, 528, 6, 1, 0, 0, 0, 0, 0, 0, 538, 545, 497, 500, 510, 527, 551, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 507, 461, 515, 517, 475, 498, 22, 0, 0, 0, 0, 0, 0, 0, 489, 490, 513, 542, 483, 481, 507, 533, 14, 0, 0, 0, 0, 0, 0, 0, 520, 515, 530, 488, 509, 523, 562, 513, 3, 1, 0, 0, 0, 0, 0, 0, 509, 505, 500, 522, 526, 482, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 518, 493, 480, 532, 548, 473, 540, 502, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4074, 4180, 4032, 4052, 4164, 4081, 4091] avg=4096.0000 max=4180 max_rank=2 min=4032 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[508, 543, 484, 564, 496, 526, 527, 483, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 519, 489, 512, 543, 535, 487, 6, 1, 0, 0, 0, 0, 0, 0, 501, 500, 475, 497, 524, 515, 499, 551, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 547, 508, 547, 526, 502, 509, 21, 0, 0, 0, 0, 0, 0, 0, 489, 485, 524, 498, 509, 525, 509, 506, 13, 0, 0, 0, 0, 0, 0, 0, 493, 467, 525, 499, 488, 538, 514, 506, 3, 1, 0, 0, 0, 0, 0, 0, 536, 514, 545, 521, 513, 484, 546, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 516, 524, 498, 513, 506, 501, 497, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4090, 4062, 4167, 4058, 4034, 4184, 4042] avg=4096.0000 max=4184 max_rank=6 min=4034 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[502, 530, 531, 514, 537, 513, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 540, 500, 518, 505, 537, 516, 499, 502, 6, 1, 0, 0, 0, 0, 0, 0, 488, 507, 504, 543, 508, 529, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 495, 477, 532, 453, 525, 511, 470, 526, 23, 0, 0, 0, 0, 0, 0, 0, 520, 506, 504, 498, 534, 522, 524, 503, 14, 0, 0, 0, 0, 0, 0, 0, 497, 494, 478, 535, 527, 505, 498, 503, 3, 1, 0, 0, 0, 0, 0, 0, 530, 551, 492, 511, 501, 521, 535, 493, 0, 0, 0, 0, 0, 0, 0, 0, 563, 500, 517, 519, 490, 508, 515, 475, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4124, 4092, 4012, 4125, 4041, 4134, 4092] avg=4096.0000 max=4148 max_rank=0 min=4012 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 484, 507, 528, 525, 512, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 478, 521, 492, 514, 516, 513, 510, 9, 3, 0, 0, 0, 0, 0, 0, 513, 544, 483, 507, 529, 514, 516, 479, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 484, 513, 506, 508, 515, 521, 27, 0, 0, 0, 0, 0, 0, 0, 539, 561, 500, 478, 508, 520, 505, 510, 17, 0, 0, 0, 0, 0, 0, 0, 523, 498, 555, 537, 523, 535, 507, 492, 7, 7, 0, 0, 0, 0, 0, 0, 489, 527, 494, 507, 509, 492, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 510, 500, 495, 503, 499, 515, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4059, 4085, 4083, 4138, 4184, 4022, 4085] avg=4096.0000 max=4184 max_rank=5 min=4022 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 529, 497, 515, 522, 491, 509, 549, 0, 0, 0, 0, 0, 0, 0, 0, 511, 542, 508, 511, 535, 518, 518, 547, 6, 1, 0, 0, 0, 0, 0, 0, 515, 521, 458, 508, 524, 516, 524, 485, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 506, 505, 527, 524, 474, 481, 21, 0, 0, 0, 0, 0, 0, 0, 512, 531, 509, 504, 475, 556, 477, 520, 13, 0, 0, 0, 0, 0, 0, 0, 505, 507, 494, 482, 499, 464, 503, 513, 3, 1, 0, 0, 0, 0, 0, 0, 503, 515, 530, 552, 505, 520, 564, 473, 0, 0, 0, 0, 0, 0, 0, 0, 479, 510, 480, 487, 535, 539, 514, 562, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4197, 4051, 4063, 4097, 3971, 4162, 4111] avg=4096.0000 max=4197 max_rank=1 min=3971 min_rank=5 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 486, 465, 509, 505, 521, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 483, 530, 490, 525, 510, 498, 6, 1, 0, 0, 0, 0, 0, 0, 521, 523, 470, 502, 536, 484, 498, 503, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 523, 544, 500, 509, 568, 531, 21, 0, 0, 0, 0, 0, 0, 0, 520, 489, 509, 494, 508, 510, 548, 512, 13, 0, 0, 0, 0, 0, 0, 0, 547, 562, 524, 525, 514, 463, 538, 510, 3, 1, 0, 0, 0, 0, 0, 0, 517, 507, 520, 500, 493, 481, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 527, 501, 517, 495, 511, 510, 539, 515, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4058, 4037, 4208, 4103, 4187, 4038, 4120] avg=4096.0000 max=4208 max_rank=3 min=4017 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[532, 493, 529, 498, 529, 494, 489, 495, 509, 532, 534, 480, 508, 514, 541, 520, 484, 534, 518, 505, 501, 509, 493, 497, 529, 526, 495, 511, 483, 494, 488, 514, 548, 502, 565, 510, 504, 492, 515, 504, 481, 501, 520, 520, 483, 511, 541, 527, 517, 507, 526, 536, 528, 537, 534, 481, 517, 512, 485, 518, 519, 523, 516, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[476, 480, 522, 547, 491, 518, 537, 531, 499, 532, 526, 499, 550, 467, 497, 546, 559, 518, 530, 514, 540, 524, 516, 534, 543, 527, 498, 486, 492, 483, 530, 547, 475, 501, 560, 530, 493, 481, 485, 494, 460, 503, 482, 530, 514, 517, 515, 494, 516, 479, 493, 520, 543, 551, 543, 532, 479, 508, 499, 530, 494, 508, 497, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[500, 489, 506, 527, 500, 492, 538, 533, 512, 507, 483, 494, 501, 473, 512, 498, 541, 465, 500, 527, 510, 513, 519, 508, 491, 519, 518, 490, 510, 543, 491, 541, 514, 486, 497, 515, 526, 529, 552, 515, 484, 539, 498, 545, 530, 538, 535, 527, 497, 502, 546, 512, 500, 523, 537, 483, 513, 523, 516, 498, 508, 491, 529, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[512, 476, 519, 511, 478, 527, 494, 508, 533, 509, 477, 492, 533, 513, 531, 508, 498, 508, 524, 499, 513, 492, 503, 524, 536, 499, 517, 505, 499, 510, 497, 517, 524, 512, 510, 518, 542, 504, 489, 491, 566, 521, 500, 556, 532, 525, 524, 502, 496, 527, 490, 525, 531, 466, 537, 536, 518, 478, 509, 488, 534, 501, 515, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[469, 477, 503, 550, 532, 511, 506, 532, 522, 479, 499, 498, 541, 523, 529, 498, 530, 471, 531, 521, 493, 541, 523, 506, 524, 516, 500, 533, 523, 534, 519, 491, 495, 539, 504, 513, 541, 460, 494, 476, 521, 496, 559, 480, 517, 512, 522, 475, 491, 508, 530, 502, 500, 550, 522, 512, 501, 496, 508, 492, 519, 558, 522, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[479, 476, 515, 534, 516, 502, 517, 479, 498, 520, 505, 518, 472, 548, 500, 519, 516, 498, 527, 506, 512, 494, 506, 569, 534, 501, 513, 535, 496, 496, 506, 504, 522, 505, 484, 525, 480, 504, 545, 532, 510, 487, 525, 532, 541, 456, 512, 527, 497, 513, 479, 510, 527, 553, 539, 516, 539, 537, 517, 484, 526, 492, 514, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[487, 508, 512, 525, 533, 512, 504, 505, 522, 509, 542, 524, 512, 514, 503, 530, 524, 518, 510, 515, 493, 509, 535, 485, 529, 511, 502, 513, 501, 528, 527, 500, 531, 508, 523, 501, 490, 521, 548, 513, 494, 500, 517, 498, 521, 535, 509, 491, 492, 492, 499, 509, 516, 495, 519, 491, 514, 528, 525, 487, 518, 499, 539, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[508, 506, 489, 521, 525, 526, 541, 535, 516, 530, 509, 506, 529, 541, 532, 455, 520, 512, 513, 521, 502, 548, 511, 505, 506, 527, 499, 538, 536, 513, 535, 501, 508, 496, 489, 496, 524, 462, 496, 508, 509, 526, 513, 528, 496, 507, 509, 490, 497, 482, 484, 509, 527, 510, 531, 504, 516, 487, 515, 524, 510, 542, 483, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[532, 493, 529, 498, 529, 494, 489, 495, 13, 5, 0, 0, 0, 0, 0, 0, 509, 532, 534, 480, 508, 514, 541, 520, 0, 0, 0, 0, 0, 0, 0, 0, 471, 534, 518, 505, 501, 509, 493, 497, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 495, 511, 483, 494, 488, 514, 0, 0, 0, 0, 0, 0, 0, 0, 548, 502, 565, 510, 504, 492, 515, 504, 8, 5, 5, 0, 0, 0, 0, 0, 481, 501, 520, 512, 483, 511, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 526, 536, 528, 537, 524, 481, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 485, 518, 519, 523, 516, 510, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4138, 4028, 4034, 4158, 4076, 4156, 4101] avg=4096.0000 max=4158 max_rank=4 min=4028 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 476, 519, 511, 478, 527, 494, 508, 13, 5, 0, 0, 0, 0, 0, 0, 533, 509, 477, 492, 533, 513, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 485, 508, 524, 499, 513, 492, 503, 524, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 517, 505, 499, 510, 497, 517, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 510, 518, 542, 504, 489, 491, 9, 5, 5, 0, 0, 0, 0, 0, 566, 521, 500, 547, 532, 525, 524, 502, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 490, 525, 531, 466, 527, 536, 0, 0, 0, 0, 0, 0, 0, 0, 518, 478, 509, 488, 534, 501, 515, 539, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4096, 4048, 4074, 4109, 4217, 4098, 4083] avg=4096.0000 max=4217 max_rank=5 min=4043 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[476, 480, 522, 547, 491, 518, 537, 531, 15, 5, 0, 0, 0, 0, 0, 0, 499, 532, 526, 499, 550, 467, 497, 546, 0, 0, 0, 0, 0, 0, 0, 0, 544, 518, 530, 514, 540, 524, 516, 534, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 498, 486, 492, 483, 530, 547, 0, 0, 0, 0, 0, 0, 0, 0, 475, 501, 560, 530, 493, 481, 485, 494, 8, 5, 5, 0, 0, 0, 0, 0, 460, 503, 482, 522, 514, 517, 515, 494, 0, 0, 0, 0, 0, 0, 0, 0, 516, 479, 493, 520, 543, 551, 533, 532, 0, 0, 0, 0, 0, 0, 0, 0, 479, 508, 499, 530, 494, 508, 497, 483, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4116, 4220, 4100, 4037, 4007, 4167, 3999] avg=4096.0000 max=4220 max_rank=2 min=3999 min_rank=7 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 489, 506, 527, 500, 492, 538, 533, 15, 5, 0, 0, 0, 0, 0, 0, 512, 507, 483, 494, 501, 473, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 526, 465, 500, 527, 510, 513, 519, 508, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 518, 490, 510, 543, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 497, 515, 526, 529, 552, 515, 8, 5, 4, 0, 0, 0, 0, 0, 484, 539, 498, 537, 530, 538, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 546, 512, 500, 523, 527, 483, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 516, 498, 508, 491, 529, 509, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3980, 4068, 4098, 4151, 4188, 4090, 4088] avg=4096.0000 max=4188 max_rank=5 min=3980 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[469, 477, 503, 550, 532, 511, 506, 532, 14, 5, 0, 0, 0, 0, 0, 0, 522, 479, 499, 498, 541, 523, 529, 498, 0, 0, 0, 0, 0, 0, 0, 0, 516, 471, 531, 521, 493, 541, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 500, 533, 523, 534, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 495, 539, 504, 513, 541, 460, 494, 476, 7, 5, 5, 0, 0, 0, 0, 0, 521, 496, 559, 473, 517, 512, 522, 475, 0, 0, 0, 0, 0, 0, 0, 0, 491, 508, 530, 502, 500, 550, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 508, 492, 519, 558, 522, 528, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4089, 4102, 4134, 4039, 4075, 4105, 4125] avg=4096.0000 max=4134 max_rank=3 min=4039 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 508, 512, 525, 533, 512, 504, 505, 19, 10, 0, 0, 0, 0, 0, 0, 522, 509, 542, 524, 507, 514, 503, 530, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 510, 515, 493, 509, 535, 485, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 502, 513, 501, 528, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 523, 501, 490, 521, 548, 513, 12, 9, 7, 0, 0, 0, 0, 0, 494, 500, 517, 486, 521, 535, 509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 492, 492, 499, 509, 516, 495, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 525, 487, 518, 499, 539, 503, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4151, 4070, 4099, 4163, 4053, 3994, 4123] avg=4096.0000 max=4163 max_rank=4 min=3994 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[479, 476, 515, 534, 516, 502, 517, 479, 14, 5, 0, 0, 0, 0, 0, 0, 498, 520, 505, 518, 472, 548, 500, 519, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 527, 506, 512, 494, 506, 569, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 513, 535, 496, 496, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 522, 505, 484, 525, 480, 504, 545, 532, 8, 5, 5, 0, 0, 0, 0, 0, 510, 487, 525, 524, 541, 456, 512, 527, 0, 0, 0, 0, 0, 0, 0, 0, 497, 513, 479, 510, 527, 553, 529, 516, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 517, 484, 526, 492, 514, 527, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4080, 4114, 4079, 4115, 4082, 4124, 4137] avg=4096.0000 max=4137 max_rank=7 min=4037 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 506, 489, 521, 525, 526, 541, 535, 14, 5, 0, 0, 0, 0, 0, 0, 516, 530, 509, 506, 529, 541, 532, 455, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 513, 521, 502, 548, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 501, 527, 499, 538, 536, 513, 535, 501, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 489, 496, 524, 462, 496, 508, 8, 5, 4, 0, 0, 0, 0, 0, 509, 526, 513, 520, 496, 507, 509, 490, 0, 0, 0, 0, 0, 0, 0, 0, 497, 482, 484, 509, 527, 510, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 516, 487, 515, 524, 510, 542, 483, 534, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4118, 4118, 4150, 3996, 4070, 4034, 4112] avg=4096.0000 max=4170 max_rank=0 min=3996 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16212 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[487, 512, 512, 510, 513, 509, 511, 520, 535, 519, 521, 565, 520, 494, 504, 534, 487, 519, 510, 526, 520, 503, 467, 497, 520, 514, 498, 489, 527, 523, 501, 525, 493, 494, 495, 538, 487, 494, 524, 528, 530, 524, 533, 507, 500, 491, 465, 557, 520, 473, 549, 518, 527, 526, 536, 522, 493, 515, 544, 506, 496, 481, 492, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[511, 499, 515, 528, 538, 507, 518, 504, 515, 539, 526, 456, 539, 522, 536, 498, 500, 490, 502, 477, 535, 490, 538, 500, 505, 482, 511, 530, 494, 551, 539, 490, 537, 544, 509, 505, 494, 467, 508, 522, 519, 506, 539, 507, 570, 533, 501, 508, 522, 525, 472, 497, 491, 476, 542, 515, 504, 488, 518, 497, 525, 533, 516, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[529, 516, 520, 495, 512, 527, 525, 478, 496, 482, 508, 484, 470, 536, 510, 495, 491, 465, 510, 500, 535, 512, 554, 474, 549, 504, 551, 532, 536, 572, 521, 514, 514, 520, 510, 521, 504, 509, 538, 512, 506, 496, 522, 507, 536, 486, 484, 536, 570, 516, 510, 508, 482, 511, 530, 462, 479, 520, 504, 544, 524, 489, 513, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[474, 532, 493, 479, 498, 487, 507, 504, 515, 563, 510, 536, 510, 525, 455, 477, 542, 541, 470, 508, 502, 511, 544, 532, 534, 510, 540, 496, 493, 524, 514, 496, 499, 530, 490, 504, 532, 525, 538, 534, 530, 523, 532, 528, 511, 493, 485, 520, 510, 516, 516, 527, 531, 529, 476, 498, 492, 494, 489, 516, 490, 530, 536, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[512, 513, 572, 513, 529, 524, 514, 516, 487, 522, 510, 508, 537, 529, 487, 501, 514, 540, 526, 508, 525, 484, 536, 543, 485, 500, 513, 505, 516, 511, 499, 547, 524, 485, 496, 504, 484, 508, 515, 505, 537, 505, 484, 532, 509, 519, 501, 550, 499, 522, 520, 488, 468, 521, 498, 495, 540, 522, 490, 496, 486, 528, 495, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[485, 484, 507, 538, 545, 505, 486, 503, 523, 458, 519, 488, 493, 500, 542, 508, 477, 505, 463, 498, 499, 502, 503, 534, 503, 551, 544, 521, 520, 492, 505, 565, 513, 534, 531, 520, 522, 518, 505, 538, 526, 483, 492, 491, 503, 503, 543, 508, 556, 489, 532, 495, 504, 492, 517, 548, 481, 523, 498, 555, 508, 524, 531, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[498, 566, 493, 515, 503, 527, 527, 520, 550, 490, 531, 532, 501, 506, 510, 499, 490, 547, 498, 474, 494, 524, 514, 506, 497, 491, 516, 573, 514, 519, 499, 482, 534, 550, 524, 503, 464, 515, 475, 515, 533, 518, 490, 501, 500, 524, 536, 482, 524, 511, 519, 508, 464, 564, 515, 510, 500, 511, 501, 513, 510, 531, 504, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[532, 535, 532, 494, 512, 541, 464, 516, 502, 509, 481, 496, 525, 526, 510, 507, 520, 493, 516, 505, 518, 489, 521, 522, 515, 545, 530, 525, 563, 511, 492, 549, 508, 525, 512, 519, 516, 485, 511, 526, 482, 518, 510, 556, 520, 509, 489, 469, 509, 547, 531, 465, 533, 510, 499, 513, 486, 502, 502, 489, 529, 505, 493, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[487, 510, 512, 510, 513, 509, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 521, 565, 520, 494, 504, 534, 4, 3, 2, 0, 0, 0, 0, 0, 487, 519, 510, 526, 520, 503, 467, 497, 28, 0, 0, 0, 0, 0, 0, 0, 520, 514, 498, 489, 527, 472, 501, 525, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 495, 538, 487, 494, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 533, 507, 500, 491, 465, 557, 0, 0, 0, 0, 0, 0, 0, 0, 520, 473, 549, 518, 527, 526, 536, 522, 0, 0, 0, 0, 0, 0, 0, 0, 493, 515, 544, 506, 496, 481, 492, 518, 23, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4201, 4057, 4046, 4049, 4103, 4171, 4069] avg=4096.0000 max=4201 max_rank=1 min=4046 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[474, 530, 493, 479, 498, 487, 507, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 563, 510, 536, 510, 525, 455, 477, 4, 3, 2, 0, 0, 0, 0, 0, 542, 541, 470, 508, 502, 511, 544, 532, 28, 0, 0, 0, 0, 0, 0, 0, 534, 510, 540, 496, 493, 473, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 499, 526, 490, 504, 532, 525, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 526, 523, 532, 528, 511, 493, 485, 520, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 516, 527, 531, 529, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 489, 516, 490, 530, 536, 522, 23, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3972, 4100, 4178, 4056, 4148, 4118, 4103, 4093] avg=4096.0000 max=4178 max_rank=2 min=3972 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[511, 497, 515, 528, 538, 507, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 526, 456, 539, 522, 536, 498, 4, 3, 2, 0, 0, 0, 0, 0, 500, 490, 502, 477, 535, 490, 538, 500, 29, 0, 0, 0, 0, 0, 0, 0, 505, 482, 511, 530, 494, 498, 539, 490, 0, 0, 0, 0, 0, 0, 0, 0, 537, 540, 509, 505, 494, 467, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 539, 507, 570, 533, 501, 508, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 472, 497, 491, 476, 542, 515, 0, 0, 0, 0, 0, 0, 0, 0, 504, 488, 518, 497, 525, 533, 516, 493, 24, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4140, 4061, 4049, 4082, 4179, 4040, 4099] avg=4096.0000 max=4179 max_rank=5 min=4040 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[485, 482, 507, 538, 545, 505, 486, 503, 0, 0, 0, 0, 0, 0, 0, 0, 523, 458, 519, 488, 493, 500, 542, 508, 4, 3, 2, 0, 0, 0, 0, 0, 477, 505, 463, 498, 499, 502, 503, 534, 26, 0, 0, 0, 0, 0, 0, 0, 503, 551, 544, 521, 520, 445, 505, 565, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 531, 520, 522, 518, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 492, 491, 503, 503, 543, 508, 0, 0, 0, 0, 0, 0, 0, 0, 556, 489, 532, 495, 504, 492, 517, 548, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 498, 555, 508, 524, 531, 519, 21, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4040, 4007, 4154, 4177, 4045, 4133, 4161] avg=4096.0000 max=4177 max_rank=4 min=4007 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 511, 572, 513, 529, 524, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 510, 508, 537, 529, 487, 501, 4, 3, 2, 0, 0, 0, 0, 0, 514, 540, 526, 508, 525, 484, 536, 543, 27, 0, 0, 0, 0, 0, 0, 0, 485, 500, 513, 505, 516, 462, 499, 547, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 496, 504, 484, 508, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 533, 505, 484, 532, 509, 519, 501, 550, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 520, 488, 468, 521, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 540, 522, 490, 496, 486, 528, 495, 516, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4090, 4203, 4027, 4017, 4133, 4011, 4096] avg=4096.0000 max=4203 max_rank=2 min=4011 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[529, 514, 520, 495, 512, 527, 525, 478, 0, 0, 0, 0, 0, 0, 0, 0, 496, 482, 508, 484, 470, 536, 510, 495, 4, 3, 2, 0, 0, 0, 0, 0, 491, 465, 510, 500, 535, 512, 554, 474, 31, 0, 0, 0, 0, 0, 0, 0, 549, 504, 551, 532, 536, 516, 521, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 510, 521, 504, 509, 538, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 496, 522, 507, 536, 486, 484, 536, 0, 0, 0, 0, 0, 0, 0, 0, 570, 516, 510, 508, 482, 511, 530, 462, 0, 0, 0, 0, 0, 0, 0, 0, 479, 520, 504, 544, 524, 489, 513, 502, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3990, 4072, 4223, 4124, 4069, 4089, 4101] avg=4096.0000 max=4223 max_rank=3 min=3990 min_rank=1 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[498, 557, 493, 515, 503, 527, 527, 520, 0, 0, 0, 0, 0, 0, 0, 0, 550, 490, 531, 532, 501, 506, 510, 499, 7, 6, 9, 1, 0, 0, 0, 0, 490, 547, 498, 474, 494, 524, 514, 506, 32, 0, 0, 0, 0, 0, 0, 0, 497, 491, 516, 573, 514, 460, 499, 482, 0, 0, 0, 0, 0, 0, 0, 0, 534, 543, 524, 503, 464, 515, 475, 515, 0, 0, 0, 0, 0, 0, 0, 0, 521, 518, 490, 501, 500, 524, 536, 482, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 519, 508, 464, 564, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 501, 513, 510, 531, 504, 513, 27, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4142, 4079, 4032, 4073, 4072, 4114, 4116] avg=4096.0000 max=4142 max_rank=1 min=4032 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[532, 533, 532, 494, 512, 541, 464, 516, 0, 0, 0, 0, 0, 0, 0, 0, 502, 509, 481, 496, 525, 526, 510, 507, 4, 3, 2, 0, 0, 0, 0, 0, 520, 493, 516, 505, 518, 489, 521, 522, 27, 0, 0, 0, 0, 0, 0, 0, 515, 545, 530, 525, 563, 462, 492, 549, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 512, 519, 516, 485, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 478, 518, 510, 556, 520, 509, 489, 469, 0, 0, 0, 0, 0, 0, 0, 0, 509, 547, 531, 465, 533, 510, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 502, 489, 529, 505, 493, 504, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4065, 4111, 4181, 4098, 4049, 4107, 4033] avg=4096.0000 max=4181 max_rank=3 min=4033 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 3840, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[550, 486, 489, 528, 506, 510, 521, 519, 536, 499, 501, 520, 502, 499, 509, 482, 499, 516, 529, 510, 495, 520, 505, 540, 504, 524, 480, 500, 518, 516, 479, 520, 476, 568, 537, 527, 545, 529, 483, 522, 501, 512, 535, 514, 522, 526, 516, 528, 480, 528, 506, 539, 534, 493, 497, 514, 510, 483, 504, 499, 497, 519, 527, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[469, 500, 514, 560, 483, 511, 497, 534, 523, 517, 491, 532, 518, 503, 508, 513, 525, 506, 518, 497, 480, 518, 508, 550, 499, 541, 523, 561, 510, 546, 475, 495, 470, 531, 500, 507, 488, 501, 480, 521, 519, 518, 459, 502, 501, 522, 510, 550, 513, 523, 528, 492, 489, 507, 495, 581, 509, 519, 515, 513, 487, 519, 566, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[511, 498, 492, 519, 495, 546, 545, 552, 498, 514, 551, 526, 511, 514, 542, 508, 480, 492, 507, 503, 547, 503, 544, 514, 477, 491, 502, 526, 470, 506, 483, 512, 498, 509, 526, 524, 515, 510, 500, 512, 510, 559, 452, 564, 509, 526, 497, 507, 498, 545, 513, 518, 490, 521, 519, 527, 522, 538, 494, 499, 505, 497, 492, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[537, 494, 570, 495, 534, 512, 546, 532, 526, 530, 485, 495, 514, 516, 481, 476, 494, 541, 467, 506, 524, 534, 513, 507, 513, 491, 533, 465, 538, 483, 534, 530, 536, 519, 524, 525, 509, 549, 532, 510, 513, 492, 500, 488, 515, 517, 535, 480, 488, 532, 507, 499, 515, 529, 512, 506, 505, 480, 482, 514, 507, 485, 533, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[501, 517, 522, 504, 497, 522, 520, 506, 494, 518, 516, 503, 508, 507, 548, 497, 499, 527, 501, 496, 548, 490, 535, 512, 515, 488, 521, 512, 514, 532, 538, 504, 502, 518, 527, 489, 509, 531, 527, 503, 505, 538, 516, 495, 508, 480, 515, 543, 489, 523, 511, 510, 529, 538, 497, 522, 515, 486, 536, 508, 495, 517, 486, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[507, 472, 518, 530, 526, 493, 521, 505, 517, 561, 556, 509, 488, 517, 492, 500, 515, 513, 514, 521, 528, 480, 503, 535, 469, 536, 526, 487, 512, 532, 493, 495, 511, 518, 530, 538, 457, 507, 522, 512, 534, 499, 500, 497, 513, 504, 518, 526, 509, 518, 501, 499, 514, 486, 533, 504, 529, 523, 519, 539, 515, 513, 497, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[489, 491, 493, 490, 495, 495, 533, 542, 500, 538, 516, 503, 505, 507, 529, 523, 514, 504, 486, 497, 528, 528, 501, 492, 530, 514, 480, 524, 548, 494, 535, 525, 501, 483, 559, 526, 531, 502, 535, 528, 500, 513, 507, 536, 553, 531, 545, 499, 483, 500, 480, 490, 521, 525, 529, 501, 492, 518, 495, 509, 535, 476, 494, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[491, 494, 505, 498, 519, 502, 512, 488, 498, 458, 511, 558, 486, 522, 541, 516, 513, 505, 480, 554, 521, 516, 477, 581, 517, 517, 485, 548, 523, 516, 532, 521, 540, 479, 524, 533, 498, 502, 504, 510, 538, 494, 512, 524, 526, 491, 448, 517, 537, 530, 511, 517, 478, 526, 518, 507, 493, 515, 499, 521, 510, 497, 536, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 498, 492, 519, 495, 546, 545, 552, 1, 0, 0, 0, 0, 0, 0, 0, 498, 514, 551, 526, 511, 514, 541, 508, 0, 0, 0, 0, 0, 0, 0, 0, 480, 492, 507, 503, 547, 503, 544, 510, 0, 0, 0, 0, 0, 0, 0, 0, 477, 491, 502, 526, 470, 506, 483, 512, 4, 0, 0, 0, 0, 0, 0, 0, 498, 509, 505, 524, 515, 510, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 559, 452, 564, 509, 526, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 498, 545, 513, 518, 490, 521, 519, 527, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 494, 499, 505, 497, 492, 493, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4163, 4086, 3971, 4073, 4118, 4131, 4067] avg=4096.0000 max=4163 max_rank=1 min=3971 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[537, 494, 570, 495, 534, 512, 546, 532, 1, 0, 0, 0, 0, 0, 0, 0, 526, 530, 485, 495, 514, 516, 480, 476, 0, 0, 0, 0, 0, 0, 0, 0, 494, 541, 467, 506, 524, 534, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 533, 465, 538, 483, 534, 530, 4, 0, 0, 0, 0, 0, 0, 0, 536, 519, 503, 525, 509, 549, 532, 510, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 500, 488, 515, 517, 535, 474, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 507, 499, 515, 529, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 505, 480, 482, 514, 507, 485, 533, 514, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4022, 4082, 4091, 4183, 4034, 4088, 4047] avg=4096.0000 max=4221 max_rank=0 min=4022 min_rank=1 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[550, 486, 489, 528, 506, 510, 521, 519, 1, 0, 0, 0, 0, 0, 0, 0, 536, 499, 501, 520, 502, 499, 508, 482, 0, 0, 0, 0, 0, 0, 0, 0, 499, 516, 529, 510, 495, 520, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 504, 524, 480, 500, 518, 516, 479, 520, 4, 0, 0, 0, 0, 0, 0, 0, 476, 568, 516, 527, 545, 529, 483, 522, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 535, 514, 522, 526, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 480, 528, 506, 539, 534, 493, 497, 514, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 504, 499, 497, 519, 527, 485, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4047, 4110, 4045, 4166, 4147, 4091, 4052] avg=4096.0000 max=4166 max_rank=4 min=4045 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[469, 500, 514, 560, 483, 511, 497, 534, 1, 0, 0, 0, 0, 0, 0, 0, 523, 517, 491, 532, 518, 503, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 525, 506, 518, 497, 480, 518, 508, 546, 0, 0, 0, 0, 0, 0, 0, 0, 499, 541, 523, 561, 510, 546, 475, 495, 4, 0, 0, 0, 0, 0, 0, 0, 470, 531, 480, 507, 488, 501, 480, 521, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 459, 502, 501, 522, 510, 543, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 528, 492, 489, 507, 495, 581, 0, 0, 0, 0, 0, 0, 0, 0, 509, 519, 515, 513, 487, 519, 566, 508, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4104, 4098, 4154, 3978, 4074, 4128, 4163] avg=4096.0000 max=4163 max_rank=7 min=3978 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 472, 518, 530, 526, 493, 521, 505, 1, 0, 0, 0, 0, 0, 0, 0, 517, 561, 556, 509, 488, 517, 491, 500, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 514, 521, 528, 480, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 469, 536, 526, 487, 512, 532, 493, 495, 4, 0, 0, 0, 0, 0, 0, 0, 511, 518, 509, 538, 457, 507, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 534, 499, 500, 497, 513, 504, 518, 519, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 501, 499, 514, 486, 533, 504, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 519, 539, 515, 513, 497, 512, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4139, 4105, 4054, 4074, 4084, 4064, 4175] avg=4096.0000 max=4175 max_rank=7 min=4054 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[489, 491, 493, 490, 495, 495, 533, 542, 1, 0, 0, 0, 0, 0, 0, 0, 500, 538, 516, 503, 505, 507, 528, 523, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 486, 497, 528, 528, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 480, 524, 548, 494, 535, 525, 4, 0, 0, 0, 0, 0, 0, 0, 501, 483, 537, 526, 531, 502, 535, 528, 0, 0, 0, 0, 0, 0, 0, 0, 500, 513, 507, 536, 553, 531, 545, 493, 0, 0, 0, 0, 0, 0, 0, 0, 483, 500, 480, 490, 521, 525, 529, 501, 0, 0, 0, 0, 0, 0, 0, 0, 492, 518, 495, 509, 535, 476, 494, 522, 22, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4120, 4046, 4154, 4143, 4178, 4029, 4069] avg=4096.0000 max=4178 max_rank=5 min=4029 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 517, 522, 504, 497, 522, 520, 506, 2, 6, 0, 0, 0, 0, 0, 0, 494, 518, 516, 503, 508, 507, 541, 497, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 501, 496, 548, 490, 535, 502, 0, 0, 0, 0, 0, 0, 0, 0, 515, 488, 521, 512, 514, 532, 538, 504, 7, 5, 0, 0, 0, 0, 0, 0, 502, 518, 503, 489, 509, 531, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 505, 538, 516, 495, 508, 480, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 511, 510, 529, 538, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 536, 508, 495, 517, 486, 488, 24, 11, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4084, 4098, 4136, 4082, 4089, 4113, 4069] avg=4096.0000 max=4136 max_rank=3 min=4069 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[491, 494, 505, 498, 519, 502, 512, 488, 1, 0, 0, 0, 0, 0, 0, 0, 498, 458, 511, 558, 486, 522, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 480, 554, 521, 516, 477, 577, 0, 0, 0, 0, 0, 0, 0, 0, 517, 517, 485, 548, 523, 516, 532, 521, 4, 0, 0, 0, 0, 0, 0, 0, 540, 479, 503, 533, 498, 502, 504, 510, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 512, 524, 526, 491, 448, 511, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 511, 517, 478, 526, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 515, 499, 521, 510, 497, 536, 528, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4089, 4143, 4163, 4069, 4044, 4124, 4126] avg=4096.0000 max=4163 max_rank=3 min=4010 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 15:43:13] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 280.4 | throughput per GPU (TFLOP/s/GPU): 440.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.531645E+00 | loss scale: 1.0 | grad norm: 3.697 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[536, 521, 515, 517, 530, 501, 507, 539, 523, 474, 476, 520, 479, 504, 516, 526, 476, 476, 509, 530, 517, 506, 514, 506, 483, 534, 499, 553, 509, 499, 519, 536, 507, 509, 528, 525, 479, 508, 490, 485, 548, 523, 485, 531, 532, 518, 560, 521, 484, 463, 567, 520, 492, 512, 503, 533, 519, 479, 468, 506, 521, 530, 533, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[499, 512, 539, 525, 515, 514, 536, 511, 518, 517, 522, 515, 523, 517, 525, 533, 544, 500, 515, 500, 501, 515, 524, 521, 462, 533, 512, 507, 485, 504, 521, 518, 525, 515, 541, 476, 491, 521, 510, 537, 495, 567, 516, 452, 523, 507, 539, 521, 481, 504, 544, 486, 507, 507, 501, 510, 499, 506, 476, 511, 504, 521, 509, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[493, 493, 501, 531, 559, 509, 507, 528, 538, 513, 514, 498, 477, 528, 485, 542, 534, 517, 535, 495, 528, 519, 532, 541, 487, 523, 499, 501, 524, 482, 512, 496, 514, 541, 505, 502, 526, 490, 473, 502, 486, 533, 534, 519, 504, 508, 486, 501, 547, 517, 521, 528, 537, 539, 487, 472, 504, 507, 501, 494, 528, 509, 506, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[523, 510, 535, 511, 482, 503, 493, 518, 515, 486, 530, 528, 504, 508, 475, 526, 518, 515, 520, 510, 512, 537, 495, 503, 556, 518, 522, 523, 498, 482, 545, 506, 471, 520, 497, 550, 497, 522, 486, 502, 479, 530, 517, 478, 534, 500, 512, 500, 509, 494, 554, 516, 547, 518, 514, 472, 499, 514, 543, 546, 497, 495, 534, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[502, 554, 506, 531, 476, 478, 520, 546, 515, 517, 516, 522, 513, 498, 522, 507, 498, 459, 500, 507, 529, 525, 486, 494, 508, 499, 534, 503, 520, 532, 491, 494, 511, 487, 516, 490, 538, 543, 530, 529, 505, 510, 533, 504, 550, 490, 484, 545, 497, 510, 516, 491, 554, 489, 505, 511, 506, 549, 517, 523, 535, 518, 484, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[487, 510, 501, 516, 517, 476, 497, 526, 488, 491, 521, 537, 513, 511, 527, 528, 585, 516, 515, 513, 543, 523, 525, 538, 507, 501, 518, 505, 524, 504, 507, 493, 540, 537, 522, 512, 520, 532, 507, 480, 514, 483, 492, 521, 503, 511, 473, 486, 528, 513, 520, 497, 470, 528, 509, 493, 505, 532, 531, 516, 501, 486, 548, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[491, 513, 529, 501, 513, 512, 525, 541, 516, 499, 501, 503, 539, 530, 500, 513, 500, 491, 537, 460, 442, 506, 512, 530, 527, 488, 509, 481, 537, 490, 509, 508, 510, 517, 483, 523, 536, 501, 554, 545, 537, 518, 542, 479, 502, 478, 507, 535, 528, 566, 477, 488, 542, 516, 483, 507, 511, 528, 487, 537, 533, 516, 499, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[451, 484, 557, 506, 518, 513, 522, 531, 508, 521, 510, 546, 532, 473, 521, 508, 489, 517, 497, 500, 514, 505, 494, 519, 518, 534, 494, 528, 516, 507, 523, 473, 496, 525, 535, 500, 513, 534, 514, 501, 516, 481, 524, 515, 573, 528, 468, 557, 520, 492, 500, 478, 542, 501, 508, 494, 522, 493, 498, 514, 523, 528, 532, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[536, 521, 515, 517, 530, 501, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 523, 474, 476, 520, 479, 504, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0, 476, 476, 509, 530, 517, 506, 514, 506, 4, 0, 0, 0, 0, 0, 0, 0, 483, 534, 499, 553, 509, 499, 519, 536, 12, 5, 0, 0, 0, 0, 0, 0, 507, 509, 528, 525, 479, 502, 490, 485, 0, 0, 0, 0, 0, 0, 0, 0, 548, 523, 485, 531, 526, 518, 560, 521, 0, 0, 0, 0, 0, 0, 0, 0, 484, 463, 567, 520, 492, 512, 503, 533, 6, 4, 1, 0, 0, 0, 0, 0, 519, 479, 468, 502, 521, 530, 533, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4014, 4038, 4149, 4025, 4212, 4085, 4091] avg=4096.0000 max=4212 max_rank=5 min=4014 min_rank=1 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 554, 506, 531, 476, 478, 520, 534, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 516, 522, 513, 498, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 498, 459, 500, 507, 529, 525, 486, 494, 3, 0, 0, 0, 0, 0, 0, 0, 508, 499, 534, 503, 520, 532, 491, 494, 12, 5, 0, 0, 0, 0, 0, 0, 511, 487, 516, 490, 538, 537, 530, 529, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 533, 504, 543, 490, 484, 545, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 516, 491, 554, 489, 505, 511, 7, 4, 1, 0, 0, 0, 0, 0, 506, 549, 517, 519, 535, 518, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4107, 4001, 4098, 4138, 4114, 4085, 4124] avg=4096.0000 max=4138 max_rank=4 min=4001 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 512, 539, 525, 515, 514, 536, 500, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 522, 515, 523, 517, 525, 529, 0, 0, 0, 0, 0, 0, 0, 0, 544, 500, 515, 500, 501, 515, 524, 521, 4, 0, 0, 0, 0, 0, 0, 0, 462, 533, 512, 507, 485, 504, 521, 518, 11, 5, 0, 0, 0, 0, 0, 0, 525, 515, 541, 476, 491, 515, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 495, 567, 516, 452, 517, 507, 539, 521, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 544, 486, 507, 507, 501, 510, 6, 4, 1, 0, 0, 0, 0, 0, 499, 506, 476, 507, 504, 521, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4166, 4124, 4058, 4110, 4114, 4051, 4005] avg=4096.0000 max=4166 max_rank=1 min=4005 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[487, 510, 501, 516, 517, 476, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 488, 491, 521, 537, 513, 511, 527, 524, 0, 0, 0, 0, 0, 0, 0, 0, 585, 516, 515, 513, 543, 523, 525, 538, 4, 0, 0, 0, 0, 0, 0, 0, 507, 501, 518, 505, 524, 504, 507, 493, 11, 5, 0, 0, 0, 0, 0, 0, 540, 537, 522, 512, 520, 526, 507, 480, 0, 0, 0, 0, 0, 0, 0, 0, 514, 483, 492, 521, 497, 511, 473, 486, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 520, 497, 470, 528, 509, 493, 6, 4, 1, 0, 0, 0, 0, 0, 505, 532, 531, 512, 501, 486, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4112, 4262, 4075, 4144, 3977, 4069, 4110] avg=4096.0000 max=4262 max_rank=2 min=3977 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 493, 501, 531, 559, 509, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 538, 513, 514, 498, 477, 528, 485, 538, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 535, 495, 528, 519, 532, 541, 4, 0, 0, 0, 0, 0, 0, 0, 487, 523, 499, 501, 524, 482, 512, 496, 11, 5, 0, 0, 0, 0, 0, 0, 514, 541, 505, 502, 526, 484, 473, 502, 0, 0, 0, 0, 0, 0, 0, 0, 486, 533, 534, 519, 498, 508, 486, 501, 0, 0, 0, 0, 0, 0, 0, 0, 547, 517, 521, 528, 537, 539, 487, 472, 6, 4, 1, 0, 0, 0, 0, 0, 504, 507, 501, 490, 528, 509, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4091, 4205, 4040, 4047, 4065, 4159, 4051] avg=4096.0000 max=4205 max_rank=2 min=4040 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 510, 535, 511, 482, 503, 493, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 530, 528, 504, 508, 475, 522, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 520, 510, 512, 537, 495, 503, 4, 0, 0, 0, 0, 0, 0, 0, 556, 518, 522, 523, 498, 482, 545, 506, 11, 5, 0, 0, 0, 0, 0, 0, 471, 520, 497, 550, 497, 516, 486, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 530, 517, 478, 528, 500, 512, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 554, 516, 547, 518, 514, 472, 6, 5, 1, 0, 0, 0, 0, 0, 499, 514, 543, 541, 497, 495, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4068, 4114, 4166, 4039, 4044, 4136, 4137] avg=4096.0000 max=4166 max_rank=3 min=4039 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[491, 513, 529, 501, 513, 512, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 501, 503, 539, 530, 500, 507, 0, 0, 0, 0, 0, 0, 0, 0, 500, 491, 537, 460, 442, 506, 512, 530, 6, 1, 0, 0, 0, 0, 0, 0, 527, 488, 509, 481, 537, 490, 509, 508, 16, 8, 0, 0, 0, 0, 0, 0, 510, 517, 483, 523, 536, 485, 554, 545, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 542, 479, 491, 478, 507, 535, 0, 0, 0, 0, 0, 0, 0, 0, 528, 566, 477, 488, 542, 516, 483, 507, 11, 10, 8, 0, 0, 0, 0, 0, 511, 528, 487, 526, 533, 516, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4095, 3985, 4073, 4153, 4087, 4136, 4130] avg=4096.0000 max=4153 max_rank=4 min=3985 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[451, 484, 557, 506, 518, 513, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 510, 546, 532, 473, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 497, 500, 514, 505, 494, 519, 4, 0, 0, 0, 0, 0, 0, 0, 518, 534, 494, 528, 516, 507, 523, 473, 11, 5, 0, 0, 0, 0, 0, 0, 496, 525, 535, 500, 513, 528, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 516, 481, 524, 515, 566, 528, 468, 557, 0, 0, 0, 0, 0, 0, 0, 0, 520, 492, 500, 478, 542, 501, 508, 494, 7, 4, 1, 0, 0, 0, 0, 0, 522, 493, 498, 510, 523, 528, 532, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4115, 4039, 4109, 4112, 4155, 4047, 4120] avg=4096.0000 max=4155 max_rank=5 min=4039 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[484, 505, 480, 529, 473, 518, 519, 530, 519, 529, 507, 540, 531, 494, 503, 538, 496, 497, 534, 496, 511, 518, 580, 548, 486, 500, 511, 513, 504, 495, 490, 506, 498, 497, 522, 519, 534, 517, 537, 515, 543, 489, 533, 489, 485, 494, 520, 518, 502, 526, 495, 506, 504, 504, 512, 511, 495, 519, 529, 529, 522, 491, 532, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[490, 536, 478, 544, 522, 525, 537, 511, 515, 520, 469, 513, 497, 486, 489, 512, 491, 522, 512, 510, 513, 558, 546, 519, 517, 520, 534, 521, 501, 511, 531, 523, 535, 526, 496, 473, 543, 543, 521, 502, 479, 496, 474, 510, 503, 496, 519, 516, 488, 514, 529, 545, 527, 461, 531, 527, 496, 449, 545, 499, 541, 532, 499, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[511, 503, 491, 538, 530, 514, 491, 524, 497, 505, 548, 508, 520, 537, 532, 511, 507, 502, 505, 507, 505, 499, 497, 484, 508, 497, 535, 533, 518, 526, 539, 512, 514, 518, 539, 520, 494, 518, 550, 489, 504, 520, 496, 512, 510, 513, 465, 526, 486, 499, 528, 511, 523, 486, 511, 498, 511, 518, 509, 500, 499, 497, 551, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[510, 504, 503, 530, 514, 522, 513, 489, 561, 484, 482, 520, 484, 527, 532, 487, 492, 478, 477, 519, 525, 492, 512, 498, 508, 526, 480, 524, 502, 525, 505, 531, 505, 496, 503, 538, 492, 544, 517, 519, 520, 508, 530, 487, 502, 526, 549, 531, 511, 533, 475, 512, 499, 539, 529, 508, 493, 551, 502, 499, 585, 516, 468, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[502, 489, 491, 503, 501, 515, 494, 488, 524, 525, 535, 540, 509, 457, 507, 514, 476, 470, 505, 527, 527, 514, 503, 505, 526, 530, 502, 516, 549, 486, 497, 509, 511, 453, 526, 533, 547, 512, 514, 501, 528, 520, 502, 536, 488, 508, 504, 499, 527, 505, 505, 526, 558, 501, 538, 546, 517, 496, 518, 502, 522, 527, 528, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[480, 497, 479, 545, 526, 514, 508, 466, 557, 526, 503, 573, 548, 543, 514, 513, 539, 537, 518, 473, 500, 553, 529, 514, 535, 509, 488, 545, 526, 477, 474, 480, 504, 496, 492, 487, 453, 490, 505, 483, 528, 497, 500, 555, 543, 511, 487, 539, 537, 562, 493, 505, 485, 540, 513, 510, 508, 498, 523, 498, 517, 506, 523, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[550, 492, 491, 536, 497, 508, 549, 517, 522, 504, 498, 501, 490, 492, 498, 516, 522, 514, 493, 546, 518, 500, 560, 517, 512, 543, 517, 498, 511, 486, 483, 474, 527, 540, 533, 514, 537, 523, 519, 520, 479, 512, 499, 494, 529, 492, 450, 486, 495, 487, 564, 496, 515, 520, 504, 507, 542, 492, 544, 509, 545, 529, 486, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[484, 511, 489, 515, 492, 545, 494, 539, 539, 494, 501, 515, 539, 500, 535, 488, 519, 522, 499, 561, 485, 524, 464, 491, 530, 514, 490, 481, 520, 537, 484, 506, 505, 514, 524, 519, 513, 497, 520, 521, 499, 533, 521, 552, 524, 505, 550, 483, 484, 522, 528, 550, 518, 484, 479, 511, 542, 519, 496, 460, 519, 540, 547, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[484, 505, 480, 529, 473, 518, 519, 530, 18, 5, 0, 0, 0, 0, 0, 0, 501, 529, 507, 540, 531, 494, 503, 538, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 534, 496, 511, 518, 580, 548, 0, 0, 0, 0, 0, 0, 0, 0, 486, 500, 511, 513, 504, 495, 490, 506, 9, 2, 0, 0, 0, 0, 0, 0, 498, 497, 522, 519, 534, 517, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0, 543, 489, 533, 489, 485, 494, 520, 518, 10, 8, 0, 0, 0, 0, 0, 0, 502, 526, 495, 497, 504, 504, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 529, 529, 509, 491, 532, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4143, 4180, 4016, 4127, 4089, 4051, 4101] avg=4096.0000 max=4180 max_rank=2 min=4016 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 536, 478, 544, 522, 525, 537, 511, 18, 5, 0, 0, 0, 0, 0, 0, 497, 520, 469, 513, 497, 486, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 512, 510, 513, 558, 546, 519, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 534, 521, 501, 511, 531, 523, 10, 2, 0, 0, 0, 0, 0, 0, 535, 526, 496, 473, 543, 543, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 496, 474, 510, 503, 496, 519, 516, 10, 8, 0, 0, 0, 0, 0, 0, 488, 514, 529, 535, 527, 461, 531, 527, 0, 0, 0, 0, 0, 0, 0, 0, 496, 449, 545, 499, 528, 532, 499, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 3983, 4171, 4170, 4127, 4011, 4112, 4028] avg=4096.0000 max=4171 max_rank=2 min=3983 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[550, 492, 491, 536, 497, 508, 549, 517, 18, 5, 0, 0, 0, 0, 0, 0, 504, 504, 498, 501, 490, 492, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 493, 546, 518, 500, 560, 517, 0, 0, 0, 0, 0, 0, 0, 0, 512, 543, 517, 498, 511, 486, 483, 474, 9, 2, 0, 0, 0, 0, 0, 0, 527, 540, 533, 514, 537, 523, 507, 520, 0, 0, 0, 0, 0, 0, 0, 0, 479, 512, 499, 494, 529, 492, 450, 486, 10, 8, 0, 0, 0, 0, 0, 0, 495, 487, 564, 487, 515, 520, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 542, 492, 544, 509, 532, 529, 486, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4003, 4170, 4035, 4201, 3959, 4079, 4158] avg=4096.0000 max=4201 max_rank=4 min=3959 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[511, 503, 491, 538, 530, 514, 491, 524, 17, 5, 0, 0, 0, 0, 0, 0, 480, 505, 548, 508, 520, 537, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 507, 502, 505, 507, 505, 499, 497, 484, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 535, 533, 518, 526, 539, 512, 9, 2, 0, 0, 0, 0, 0, 0, 514, 518, 539, 520, 494, 518, 538, 489, 0, 0, 0, 0, 0, 0, 0, 0, 504, 520, 496, 512, 510, 513, 465, 526, 10, 8, 0, 0, 0, 0, 0, 0, 486, 499, 528, 502, 523, 486, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 509, 500, 486, 497, 551, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4141, 4006, 4179, 4130, 4064, 4033, 4091] avg=4096.0000 max=4179 max_rank=3 min=4006 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 504, 503, 530, 514, 522, 513, 489, 19, 6, 0, 0, 0, 0, 0, 0, 542, 484, 482, 520, 484, 527, 532, 487, 0, 0, 0, 0, 0, 0, 0, 0, 492, 478, 477, 519, 525, 492, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 480, 524, 502, 525, 505, 531, 9, 2, 0, 0, 0, 0, 0, 0, 505, 496, 503, 538, 492, 544, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 530, 487, 502, 526, 549, 531, 10, 9, 0, 0, 0, 0, 0, 0, 511, 533, 475, 503, 499, 539, 529, 508, 0, 0, 0, 0, 0, 0, 0, 0, 493, 551, 502, 499, 570, 516, 468, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4058, 3993, 4112, 4102, 4172, 4097, 4124] avg=4096.0000 max=4172 max_rank=5 min=3993 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 489, 491, 503, 501, 515, 494, 488, 22, 8, 0, 0, 0, 0, 0, 0, 502, 525, 535, 540, 509, 457, 507, 514, 0, 0, 0, 0, 0, 0, 0, 0, 476, 470, 505, 527, 527, 514, 496, 505, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 502, 516, 549, 486, 497, 509, 12, 3, 7, 0, 0, 0, 0, 0, 511, 453, 526, 533, 547, 512, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 528, 520, 502, 536, 488, 508, 504, 499, 13, 13, 0, 0, 0, 0, 0, 0, 527, 505, 505, 514, 558, 501, 538, 546, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 518, 502, 501, 527, 528, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4089, 4020, 4137, 4081, 4111, 4194, 4123] avg=4096.0000 max=4194 max_rank=6 min=4013 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 497, 479, 545, 526, 514, 508, 466, 19, 5, 0, 0, 0, 0, 0, 0, 538, 526, 503, 573, 548, 543, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 518, 473, 500, 553, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 488, 545, 526, 477, 474, 480, 9, 2, 0, 0, 0, 0, 0, 0, 504, 496, 492, 487, 453, 490, 494, 483, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 500, 555, 543, 511, 487, 539, 9, 8, 0, 0, 0, 0, 0, 0, 537, 562, 493, 496, 485, 540, 513, 510, 0, 0, 0, 0, 0, 0, 0, 0, 508, 498, 523, 498, 504, 506, 523, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4258, 4163, 4045, 3899, 4177, 4136, 4051] avg=4096.0000 max=4258 max_rank=1 min=3899 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[484, 511, 489, 515, 492, 545, 494, 539, 18, 5, 0, 0, 0, 0, 0, 0, 521, 494, 501, 515, 539, 500, 535, 488, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 499, 561, 485, 524, 464, 491, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 490, 481, 520, 537, 484, 506, 10, 2, 0, 0, 0, 0, 0, 0, 505, 514, 524, 519, 513, 497, 508, 521, 0, 0, 0, 0, 0, 0, 0, 0, 499, 533, 521, 552, 524, 505, 550, 483, 10, 8, 0, 0, 0, 0, 0, 0, 484, 522, 528, 540, 518, 484, 479, 511, 0, 0, 0, 0, 0, 0, 0, 0, 542, 519, 496, 460, 506, 540, 547, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4093, 4065, 4074, 4101, 4185, 4066, 4092] avg=4096.0000 max=4185 max_rank=5 min=4065 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 3968, 4352, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[507, 515, 492, 515, 520, 485, 482, 509, 525, 514, 546, 528, 486, 483, 519, 503, 517, 493, 501, 484, 522, 535, 525, 520, 517, 510, 484, 519, 505, 538, 536, 499, 534, 529, 505, 466, 499, 541, 547, 544, 513, 493, 531, 521, 494, 505, 530, 544, 533, 480, 488, 513, 550, 475, 493, 506, 510, 524, 484, 513, 499, 510, 520, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[507, 531, 488, 498, 492, 474, 497, 517, 528, 534, 488, 488, 502, 532, 522, 517, 492, 489, 505, 518, 478, 497, 514, 488, 524, 546, 481, 529, 560, 528, 514, 511, 529, 503, 514, 525, 515, 517, 543, 507, 485, 502, 500, 519, 512, 494, 493, 522, 532, 499, 546, 524, 494, 510, 512, 529, 533, 493, 511, 510, 532, 517, 520, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[533, 491, 498, 469, 496, 505, 508, 515, 562, 500, 526, 512, 548, 532, 520, 478, 495, 534, 500, 522, 498, 513, 518, 532, 493, 500, 533, 512, 518, 550, 494, 531, 510, 501, 524, 510, 495, 530, 495, 507, 503, 523, 548, 508, 532, 504, 505, 537, 528, 501, 507, 486, 526, 503, 501, 480, 511, 500, 461, 513, 519, 540, 514, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[492, 554, 532, 518, 531, 508, 457, 501, 500, 510, 517, 518, 489, 536, 538, 513, 506, 491, 537, 513, 534, 487, 503, 517, 526, 499, 499, 493, 482, 503, 514, 490, 492, 497, 527, 529, 498, 488, 558, 497, 554, 521, 512, 462, 515, 491, 517, 545, 528, 540, 555, 546, 517, 525, 490, 487, 486, 497, 517, 524, 534, 511, 513, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[477, 505, 547, 479, 526, 502, 470, 530, 557, 503, 520, 480, 525, 474, 564, 513, 527, 529, 501, 530, 509, 504, 512, 499, 490, 516, 508, 503, 510, 529, 498, 522, 523, 526, 490, 537, 546, 510, 533, 474, 528, 490, 516, 498, 510, 513, 505, 517, 501, 526, 517, 499, 489, 479, 525, 532, 533, 523, 488, 492, 549, 536, 508, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[474, 528, 488, 480, 517, 534, 527, 496, 481, 546, 509, 500, 547, 513, 486, 467, 514, 536, 515, 509, 507, 499, 538, 510, 476, 509, 519, 529, 500, 502, 493, 505, 519, 509, 505, 529, 517, 503, 486, 486, 476, 495, 475, 518, 540, 547, 526, 545, 549, 532, 497, 527, 575, 496, 501, 498, 505, 510, 520, 542, 516, 500, 569, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[494, 501, 488, 493, 531, 506, 521, 539, 521, 475, 533, 487, 479, 516, 476, 496, 530, 527, 494, 502, 503, 488, 501, 504, 542, 504, 531, 496, 533, 480, 537, 516, 517, 549, 508, 491, 526, 496, 515, 501, 546, 541, 507, 546, 506, 509, 534, 494, 498, 483, 510, 499, 505, 511, 551, 522, 513, 494, 530, 528, 515, 545, 528, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[511, 519, 542, 507, 514, 494, 495, 520, 505, 500, 514, 495, 501, 519, 518, 518, 487, 506, 519, 468, 500, 517, 466, 511, 531, 536, 485, 512, 516, 513, 498, 541, 546, 515, 508, 522, 499, 544, 514, 491, 480, 539, 518, 488, 495, 523, 499, 535, 505, 499, 526, 500, 512, 528, 555, 532, 518, 530, 534, 504, 508, 520, 490, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[533, 491, 498, 469, 496, 505, 508, 515, 24, 16, 5, 0, 0, 0, 0, 0, 562, 500, 526, 512, 548, 532, 520, 478, 2, 0, 0, 0, 0, 0, 0, 0, 495, 534, 500, 522, 498, 513, 518, 532, 14, 10, 4, 0, 0, 0, 0, 0, 493, 500, 533, 512, 518, 544, 494, 531, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 524, 510, 495, 530, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 548, 508, 532, 504, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 507, 486, 526, 503, 501, 480, 0, 0, 0, 0, 0, 0, 0, 0, 511, 500, 461, 513, 519, 516, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4180, 4140, 4125, 4056, 4145, 4018, 4044] avg=4096.0000 max=4180 max_rank=1 min=4018 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 492, 515, 520, 485, 482, 509, 22, 18, 6, 0, 0, 0, 0, 0, 525, 514, 546, 528, 486, 483, 519, 503, 2, 0, 0, 0, 0, 0, 0, 0, 517, 493, 501, 484, 522, 535, 525, 520, 14, 10, 4, 0, 0, 0, 0, 0, 517, 510, 484, 519, 505, 532, 536, 499, 0, 0, 0, 0, 0, 0, 0, 0, 534, 529, 505, 466, 499, 541, 529, 544, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 531, 521, 494, 505, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 519, 480, 488, 513, 550, 475, 493, 506, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 484, 513, 499, 488, 520, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4106, 4125, 4102, 4147, 4115, 4024, 4078] avg=4096.0000 max=4147 max_rank=4 min=4024 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 554, 532, 518, 531, 508, 457, 501, 22, 19, 6, 0, 0, 0, 0, 0, 500, 510, 517, 518, 489, 536, 538, 513, 1, 0, 0, 0, 0, 0, 0, 0, 506, 491, 537, 513, 534, 487, 503, 517, 14, 10, 4, 0, 0, 0, 0, 0, 526, 499, 499, 493, 482, 498, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 527, 529, 498, 488, 539, 497, 0, 0, 0, 0, 0, 0, 0, 0, 554, 521, 512, 462, 515, 491, 517, 529, 0, 0, 0, 0, 0, 0, 0, 0, 514, 540, 555, 546, 517, 525, 490, 487, 0, 0, 0, 0, 0, 0, 0, 0, 486, 497, 517, 524, 534, 489, 513, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4122, 4116, 4001, 4067, 4101, 4174, 4047] avg=4096.0000 max=4174 max_rank=6 min=4001 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 531, 488, 498, 492, 474, 497, 517, 23, 18, 5, 0, 0, 0, 0, 0, 528, 534, 488, 488, 502, 532, 522, 517, 2, 0, 0, 0, 0, 0, 0, 0, 492, 489, 505, 518, 478, 497, 514, 488, 14, 9, 4, 0, 0, 0, 0, 0, 524, 546, 481, 529, 560, 522, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 529, 503, 514, 525, 515, 517, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 485, 502, 500, 519, 512, 494, 493, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 546, 524, 494, 510, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 511, 510, 532, 494, 520, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4113, 4008, 4187, 4135, 4013, 4132, 4130] avg=4096.0000 max=4187 max_rank=3 min=4008 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[474, 528, 488, 480, 517, 534, 527, 496, 22, 16, 6, 0, 0, 0, 0, 0, 481, 546, 509, 500, 547, 513, 486, 467, 1, 0, 0, 0, 0, 0, 0, 0, 514, 536, 515, 509, 507, 499, 538, 510, 15, 10, 4, 0, 0, 0, 0, 0, 476, 509, 519, 529, 500, 497, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 509, 505, 529, 517, 503, 470, 486, 0, 0, 0, 0, 0, 0, 0, 0, 476, 495, 475, 518, 540, 547, 526, 529, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 497, 527, 575, 496, 501, 498, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 520, 542, 516, 478, 569, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4050, 4157, 4028, 4038, 4106, 4160, 4141] avg=4096.0000 max=4160 max_rank=6 min=4028 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 505, 547, 479, 526, 502, 470, 530, 23, 18, 5, 0, 0, 0, 0, 0, 557, 503, 520, 480, 525, 474, 564, 513, 2, 0, 0, 0, 0, 0, 0, 0, 527, 529, 501, 530, 509, 504, 512, 499, 13, 9, 4, 0, 0, 0, 0, 0, 490, 516, 508, 503, 510, 523, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 490, 537, 546, 510, 515, 474, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 516, 498, 510, 513, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 488, 526, 517, 499, 489, 479, 525, 532, 0, 0, 0, 0, 0, 0, 0, 0, 533, 523, 488, 492, 549, 513, 508, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4138, 4137, 4070, 4121, 4063, 4055, 4102] avg=4096.0000 max=4138 max_rank=1 min=4055 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 519, 542, 507, 514, 494, 495, 520, 26, 21, 9, 0, 0, 0, 0, 0, 505, 500, 514, 495, 501, 519, 518, 518, 5, 0, 0, 0, 0, 0, 0, 0, 487, 506, 519, 468, 500, 517, 466, 511, 18, 12, 7, 0, 0, 0, 0, 0, 531, 536, 485, 512, 516, 501, 498, 541, 0, 0, 0, 0, 0, 0, 0, 0, 546, 515, 508, 522, 499, 544, 493, 491, 0, 0, 0, 0, 0, 0, 0, 0, 480, 539, 518, 488, 495, 523, 499, 514, 0, 0, 0, 0, 0, 0, 0, 0, 487, 499, 526, 500, 512, 528, 555, 532, 0, 0, 0, 0, 0, 0, 0, 0, 518, 530, 534, 504, 508, 494, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4075, 4011, 4120, 4118, 4056, 4139, 4091] avg=4096.0000 max=4158 max_rank=0 min=4011 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 501, 488, 493, 531, 506, 521, 539, 24, 17, 5, 0, 0, 0, 0, 0, 521, 475, 533, 487, 479, 516, 476, 496, 1, 0, 0, 0, 0, 0, 0, 0, 530, 527, 494, 502, 503, 488, 501, 504, 13, 9, 3, 0, 0, 0, 0, 0, 542, 504, 531, 496, 533, 476, 537, 516, 0, 0, 0, 0, 0, 0, 0, 0, 517, 549, 508, 491, 526, 496, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 546, 541, 507, 546, 506, 509, 534, 480, 0, 0, 0, 0, 0, 0, 0, 0, 485, 483, 510, 499, 505, 511, 551, 522, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 530, 528, 515, 521, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3984, 4074, 4135, 4086, 4169, 4066, 4135] avg=4096.0000 max=4169 max_rank=5 min=3984 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4096, 3968, 4224, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[509, 502, 526, 494, 542, 503, 485, 492, 521, 522, 505, 483, 487, 493, 523, 499, 516, 519, 524, 510, 518, 498, 535, 508, 535, 510, 500, 523, 527, 498, 548, 516, 508, 486, 502, 494, 517, 492, 534, 506, 510, 552, 501, 530, 517, 510, 525, 488, 539, 501, 508, 488, 557, 496, 503, 495, 561, 479, 516, 521, 497, 526, 517, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[520, 516, 525, 507, 501, 550, 547, 504, 514, 507, 509, 529, 516, 513, 515, 532, 483, 498, 521, 487, 535, 499, 545, 508, 525, 553, 527, 515, 531, 494, 453, 515, 500, 498, 496, 527, 530, 498, 544, 509, 542, 530, 486, 446, 521, 491, 512, 508, 522, 470, 526, 484, 501, 534, 536, 542, 482, 501, 551, 493, 487, 516, 494, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[516, 535, 495, 497, 541, 491, 503, 545, 500, 529, 532, 519, 508, 524, 506, 513, 454, 523, 516, 539, 527, 509, 484, 539, 539, 513, 485, 487, 523, 508, 505, 514, 502, 513, 505, 521, 540, 528, 540, 496, 485, 544, 507, 518, 497, 550, 493, 555, 508, 477, 513, 496, 511, 534, 487, 509, 508, 495, 522, 500, 484, 490, 538, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[498, 531, 483, 507, 502, 519, 525, 486, 507, 523, 508, 490, 499, 533, 523, 493, 528, 534, 536, 483, 544, 476, 546, 536, 502, 500, 540, 517, 519, 511, 508, 495, 510, 538, 505, 494, 513, 506, 512, 531, 534, 546, 485, 478, 528, 524, 469, 499, 538, 477, 542, 546, 550, 510, 486, 513, 504, 520, 531, 556, 476, 481, 467, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[576, 536, 481, 460, 511, 506, 489, 526, 513, 552, 499, 474, 485, 514, 535, 500, 533, 500, 479, 537, 486, 480, 517, 514, 528, 526, 510, 513, 490, 542, 507, 491, 532, 493, 527, 508, 558, 485, 507, 505, 508, 531, 506, 509, 501, 537, 521, 530, 532, 519, 515, 537, 507, 526, 511, 519, 498, 478, 517, 488, 516, 516, 529, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[510, 534, 529, 513, 552, 531, 497, 497, 501, 537, 481, 540, 479, 514, 521, 496, 531, 540, 499, 537, 481, 532, 500, 514, 501, 515, 502, 510, 461, 549, 526, 491, 542, 532, 521, 493, 536, 507, 481, 470, 486, 512, 505, 496, 481, 560, 534, 501, 473, 511, 519, 497, 491, 530, 523, 502, 518, 510, 515, 532, 549, 511, 503, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[503, 543, 510, 505, 521, 469, 515, 502, 498, 529, 533, 502, 524, 520, 546, 507, 531, 546, 525, 484, 530, 525, 522, 513, 494, 483, 542, 518, 538, 488, 487, 522, 522, 496, 502, 475, 509, 493, 535, 459, 494, 535, 493, 503, 515, 519, 516, 525, 496, 523, 520, 541, 511, 470, 525, 498, 520, 511, 508, 544, 492, 555, 472, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[505, 515, 535, 534, 517, 532, 523, 473, 505, 494, 505, 536, 520, 513, 510, 518, 526, 543, 491, 520, 536, 510, 539, 513, 471, 527, 492, 491, 499, 540, 510, 509, 536, 525, 521, 460, 502, 519, 492, 514, 514, 496, 509, 514, 510, 478, 496, 519, 531, 530, 530, 487, 514, 511, 512, 506, 504, 510, 493, 491, 529, 503, 516, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[498, 522, 483, 507, 502, 519, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 508, 490, 499, 533, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 528, 504, 536, 483, 544, 476, 546, 536, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 540, 517, 519, 511, 508, 495, 4, 2, 0, 0, 0, 0, 0, 0, 510, 538, 505, 494, 513, 506, 512, 531, 9, 5, 0, 0, 0, 0, 0, 0, 534, 546, 485, 478, 528, 524, 469, 499, 3, 0, 0, 0, 0, 0, 0, 0, 538, 477, 537, 546, 550, 510, 486, 513, 0, 0, 0, 0, 0, 0, 0, 0, 504, 520, 531, 556, 476, 481, 467, 497, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4076, 4153, 4098, 4123, 4066, 4157, 4053] avg=4096.0000 max=4157 max_rank=6 min=4042 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 526, 495, 497, 541, 491, 503, 545, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 532, 519, 508, 524, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 454, 493, 516, 539, 527, 509, 484, 539, 0, 0, 0, 0, 0, 0, 0, 0, 539, 513, 485, 487, 523, 508, 505, 514, 4, 2, 0, 0, 0, 0, 0, 0, 502, 513, 505, 521, 540, 528, 540, 496, 9, 5, 0, 0, 0, 0, 0, 0, 485, 544, 507, 518, 497, 550, 493, 555, 3, 1, 0, 0, 0, 0, 0, 0, 508, 477, 508, 496, 511, 534, 487, 509, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 522, 500, 484, 490, 538, 483, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4130, 4061, 4080, 4159, 4153, 4030, 4041] avg=4096.0000 max=4159 max_rank=4 min=4030 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 507, 525, 507, 501, 550, 547, 504, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 509, 529, 516, 513, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 483, 470, 521, 487, 535, 499, 545, 508, 0, 0, 0, 0, 0, 0, 0, 0, 525, 553, 527, 515, 531, 494, 453, 515, 4, 2, 0, 0, 0, 0, 0, 0, 500, 498, 496, 527, 530, 498, 544, 509, 8, 5, 0, 0, 0, 0, 0, 0, 542, 530, 486, 446, 521, 491, 512, 508, 3, 0, 0, 0, 0, 0, 0, 0, 522, 470, 521, 484, 501, 534, 536, 542, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 551, 493, 487, 516, 494, 497, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4135, 4048, 4119, 4115, 4039, 4110, 4041] avg=4096.0000 max=4161 max_rank=0 min=4039 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[510, 525, 529, 513, 552, 531, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 481, 540, 479, 514, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 499, 537, 481, 532, 500, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 502, 510, 461, 549, 526, 491, 4, 2, 0, 0, 0, 0, 0, 0, 542, 532, 521, 493, 536, 507, 481, 470, 9, 5, 0, 0, 0, 0, 0, 0, 486, 512, 505, 496, 481, 560, 534, 501, 3, 1, 0, 0, 0, 0, 0, 0, 473, 511, 514, 497, 491, 530, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 515, 532, 549, 511, 503, 506, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4068, 4104, 4061, 4096, 4079, 4041, 4165] avg=4096.0000 max=4165 max_rank=7 min=4041 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[576, 520, 481, 460, 511, 506, 489, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 547, 499, 474, 485, 514, 535, 500, 0, 0, 0, 0, 0, 0, 0, 0, 533, 466, 479, 537, 486, 480, 517, 514, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 510, 513, 490, 542, 507, 491, 8, 9, 0, 0, 0, 0, 0, 0, 532, 493, 527, 508, 558, 485, 507, 505, 11, 8, 0, 0, 0, 0, 0, 0, 508, 531, 506, 509, 501, 537, 521, 530, 4, 5, 0, 0, 0, 0, 0, 0, 532, 519, 502, 537, 507, 526, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 498, 478, 517, 488, 516, 516, 529, 492, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4067, 4012, 4124, 4134, 4152, 4153, 4057] avg=4096.0000 max=4153 max_rank=6 min=4012 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 493, 526, 494, 542, 503, 485, 492, 0, 0, 0, 0, 0, 0, 0, 0, 521, 522, 505, 483, 487, 493, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 524, 510, 518, 498, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 500, 523, 527, 498, 548, 516, 4, 2, 0, 0, 0, 0, 0, 0, 508, 486, 502, 494, 517, 492, 534, 506, 9, 5, 0, 0, 0, 0, 0, 0, 510, 552, 501, 530, 517, 510, 525, 488, 3, 0, 0, 0, 0, 0, 0, 0, 539, 501, 503, 488, 557, 496, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 561, 479, 516, 521, 497, 526, 517, 521, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4033, 4098, 4163, 4053, 4136, 4082, 4159] avg=4096.0000 max=4163 max_rank=3 min=4033 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[503, 534, 510, 505, 521, 469, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 498, 528, 533, 502, 524, 520, 546, 507, 0, 0, 0, 0, 0, 0, 0, 0, 531, 515, 525, 484, 530, 525, 522, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 483, 542, 518, 538, 488, 487, 522, 4, 2, 0, 0, 0, 0, 0, 0, 522, 496, 502, 475, 509, 493, 535, 459, 9, 5, 0, 0, 0, 0, 0, 0, 494, 535, 493, 503, 515, 519, 516, 525, 3, 1, 0, 0, 0, 0, 0, 0, 496, 523, 515, 541, 511, 470, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 520, 511, 508, 544, 492, 555, 472, 516, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4158, 4145, 4078, 4005, 4104, 4079, 4140] avg=4096.0000 max=4158 max_rank=1 min=4005 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[505, 506, 535, 534, 517, 532, 523, 473, 0, 0, 0, 0, 0, 0, 0, 0, 505, 494, 505, 536, 520, 513, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 526, 512, 491, 520, 536, 510, 539, 513, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 492, 491, 499, 540, 510, 509, 4, 2, 0, 0, 0, 0, 0, 0, 536, 525, 521, 460, 502, 519, 492, 514, 9, 5, 0, 0, 0, 0, 0, 0, 514, 496, 509, 514, 510, 478, 496, 519, 3, 0, 0, 0, 0, 0, 0, 0, 531, 530, 525, 487, 514, 511, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 493, 491, 529, 503, 516, 544, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4101, 4147, 4045, 4083, 4039, 4116, 4112] avg=4096.0000 max=4147 max_rank=2 min=4039 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4352, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[513, 527, 487, 506, 482, 503, 513, 501, 531, 509, 488, 508, 543, 510, 512, 511, 500, 533, 497, 499, 527, 501, 541, 511, 529, 516, 522, 532, 531, 503, 518, 521, 489, 495, 515, 517, 474, 505, 553, 500, 499, 487, 480, 504, 512, 493, 526, 533, 531, 503, 526, 533, 513, 516, 493, 529, 515, 517, 495, 501, 544, 510, 507, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[540, 546, 496, 522, 463, 542, 549, 481, 512, 500, 495, 560, 532, 513, 548, 491, 515, 531, 499, 474, 525, 527, 496, 511, 546, 477, 513, 507, 524, 482, 542, 500, 528, 517, 504, 488, 573, 505, 504, 509, 489, 503, 487, 527, 509, 516, 511, 501, 523, 553, 466, 480, 523, 501, 555, 494, 505, 501, 529, 491, 506, 509, 513, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[527, 501, 526, 489, 483, 518, 509, 571, 515, 513, 520, 480, 533, 522, 506, 487, 506, 530, 527, 529, 523, 494, 514, 510, 513, 485, 507, 517, 510, 509, 520, 467, 536, 526, 514, 502, 517, 499, 515, 490, 530, 500, 497, 521, 522, 530, 516, 500, 508, 502, 489, 499, 564, 508, 506, 527, 521, 509, 535, 502, 471, 492, 559, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[489, 528, 507, 505, 522, 506, 554, 495, 529, 551, 509, 493, 524, 515, 508, 462, 542, 486, 528, 518, 495, 491, 481, 499, 560, 521, 511, 498, 511, 514, 509, 507, 506, 519, 522, 546, 473, 480, 522, 495, 508, 509, 481, 521, 533, 503, 556, 527, 513, 526, 544, 473, 524, 530, 524, 497, 512, 490, 516, 535, 500, 520, 518, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[496, 514, 512, 534, 490, 518, 516, 499, 478, 503, 475, 524, 510, 536, 503, 522, 510, 491, 481, 500, 479, 536, 527, 503, 511, 532, 489, 511, 525, 520, 536, 530, 522, 513, 497, 513, 492, 525, 467, 516, 524, 552, 481, 525, 523, 508, 491, 507, 504, 512, 490, 492, 575, 536, 555, 499, 522, 556, 516, 552, 484, 507, 496, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[489, 490, 507, 501, 494, 519, 479, 514, 565, 482, 527, 498, 508, 466, 551, 565, 471, 535, 507, 506, 538, 516, 494, 530, 524, 484, 525, 503, 493, 514, 532, 529, 506, 514, 525, 499, 496, 531, 516, 532, 505, 500, 538, 549, 501, 523, 495, 515, 494, 503, 499, 479, 477, 496, 508, 539, 505, 515, 530, 510, 507, 549, 513, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[520, 526, 533, 487, 495, 491, 514, 524, 483, 519, 536, 496, 511, 517, 481, 529, 533, 488, 506, 497, 497, 544, 505, 506, 505, 516, 529, 487, 514, 518, 529, 536, 524, 477, 546, 525, 530, 468, 484, 492, 473, 504, 493, 499, 511, 502, 506, 516, 530, 515, 517, 538, 495, 538, 542, 474, 514, 542, 517, 528, 514, 518, 543, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[488, 492, 529, 527, 550, 495, 470, 495, 502, 529, 490, 522, 488, 527, 505, 488, 475, 497, 490, 504, 501, 518, 520, 493, 531, 514, 543, 511, 510, 497, 492, 512, 528, 497, 506, 584, 505, 542, 530, 517, 540, 500, 492, 494, 537, 507, 538, 518, 511, 534, 545, 515, 532, 529, 498, 485, 521, 516, 509, 482, 474, 510, 515, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 527, 487, 506, 482, 503, 513, 501, 13, 6, 0, 0, 0, 0, 0, 0, 531, 509, 488, 508, 543, 510, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 497, 499, 527, 501, 541, 511, 19, 6, 0, 0, 0, 0, 0, 0, 510, 516, 522, 532, 531, 503, 518, 521, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 515, 517, 474, 505, 553, 500, 1, 0, 0, 0, 0, 0, 0, 0, 499, 487, 480, 504, 512, 493, 526, 533, 8, 0, 0, 0, 0, 0, 0, 0, 531, 503, 526, 533, 494, 516, 493, 529, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 495, 501, 544, 510, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4112, 4134, 4153, 4049, 4042, 4125, 4102] avg=4096.0000 max=4153 max_rank=3 min=4042 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[540, 546, 496, 522, 463, 542, 549, 481, 13, 6, 0, 0, 0, 0, 0, 0, 512, 500, 495, 560, 532, 513, 548, 491, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 499, 474, 525, 527, 496, 511, 20, 6, 0, 0, 0, 0, 0, 0, 527, 477, 513, 507, 524, 482, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 504, 488, 573, 505, 504, 509, 1, 0, 0, 0, 0, 0, 0, 0, 489, 503, 487, 527, 509, 516, 511, 501, 8, 0, 0, 0, 0, 0, 0, 0, 523, 553, 466, 480, 503, 501, 555, 494, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 529, 491, 506, 509, 498, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4151, 4104, 4072, 4129, 4051, 4075, 4028] avg=4096.0000 max=4158 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[489, 528, 507, 505, 522, 506, 554, 495, 14, 6, 0, 0, 0, 0, 0, 0, 529, 551, 509, 493, 524, 515, 508, 462, 0, 0, 0, 0, 0, 0, 0, 0, 542, 486, 528, 518, 495, 491, 481, 499, 20, 6, 0, 0, 0, 0, 0, 0, 540, 521, 511, 498, 511, 514, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 522, 546, 473, 480, 522, 495, 1, 0, 0, 0, 0, 0, 0, 0, 508, 509, 481, 521, 533, 503, 556, 527, 8, 0, 0, 0, 0, 0, 0, 0, 513, 526, 544, 473, 504, 530, 524, 497, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 516, 535, 500, 520, 503, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4091, 4066, 4111, 4064, 4146, 4111, 4053] avg=4096.0000 max=4146 max_rank=5 min=4053 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[527, 501, 526, 489, 483, 518, 509, 571, 13, 6, 0, 0, 0, 0, 0, 0, 515, 513, 520, 480, 533, 522, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 527, 529, 523, 494, 514, 510, 21, 5, 0, 0, 0, 0, 0, 0, 495, 485, 507, 517, 510, 509, 520, 467, 0, 0, 0, 0, 0, 0, 0, 0, 536, 526, 514, 502, 517, 499, 515, 490, 1, 0, 0, 0, 0, 0, 0, 0, 530, 500, 497, 521, 522, 530, 516, 500, 9, 0, 0, 0, 0, 0, 0, 0, 508, 502, 489, 499, 543, 508, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 521, 509, 535, 502, 471, 492, 543, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4076, 4159, 4010, 4100, 4125, 4082, 4073] avg=4096.0000 max=4159 max_rank=2 min=4010 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 514, 512, 534, 490, 518, 516, 499, 12, 6, 0, 0, 0, 0, 0, 0, 478, 503, 475, 524, 510, 536, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 481, 500, 479, 536, 527, 503, 22, 5, 0, 0, 0, 0, 0, 0, 494, 532, 489, 511, 525, 520, 536, 530, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 497, 513, 492, 525, 467, 516, 1, 0, 0, 0, 0, 0, 0, 0, 524, 552, 481, 525, 523, 508, 491, 507, 8, 0, 0, 0, 0, 0, 0, 0, 504, 512, 490, 492, 553, 536, 555, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 556, 516, 552, 484, 507, 481, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4051, 4054, 4137, 4046, 4119, 4141, 4123] avg=4096.0000 max=4141 max_rank=6 min=4046 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[489, 490, 507, 501, 494, 519, 479, 514, 13, 6, 0, 0, 0, 0, 0, 0, 565, 482, 527, 498, 508, 466, 551, 565, 0, 0, 0, 0, 0, 0, 0, 0, 471, 535, 507, 506, 538, 516, 494, 530, 18, 6, 0, 0, 0, 0, 0, 0, 505, 484, 525, 503, 493, 514, 532, 529, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 525, 499, 496, 531, 516, 532, 1, 0, 0, 0, 0, 0, 0, 0, 505, 500, 538, 549, 501, 523, 495, 515, 8, 0, 0, 0, 0, 0, 0, 0, 494, 503, 499, 479, 459, 496, 508, 539, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 530, 510, 507, 549, 498, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4162, 4121, 4085, 4120, 4134, 3977, 4157] avg=4096.0000 max=4162 max_rank=1 min=3977 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[488, 492, 529, 527, 550, 495, 470, 495, 13, 6, 0, 0, 0, 0, 0, 0, 502, 529, 490, 522, 488, 527, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 475, 497, 490, 504, 501, 518, 520, 493, 20, 6, 0, 0, 0, 0, 0, 0, 512, 514, 543, 511, 510, 497, 492, 512, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 506, 584, 505, 542, 530, 517, 1, 0, 0, 0, 0, 0, 0, 0, 540, 500, 492, 494, 537, 507, 538, 518, 8, 0, 0, 0, 0, 0, 0, 0, 511, 534, 545, 515, 512, 529, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 509, 482, 474, 510, 500, 552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4051, 4024, 4091, 4210, 4134, 4129, 4064] avg=4096.0000 max=4210 max_rank=4 min=4024 min_rank=2 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[520, 526, 533, 487, 495, 491, 514, 524, 16, 10, 0, 0, 0, 0, 0, 0, 483, 519, 536, 496, 511, 517, 481, 529, 2, 0, 0, 0, 0, 0, 0, 0, 533, 488, 506, 497, 497, 544, 505, 506, 21, 9, 0, 0, 0, 0, 0, 0, 480, 516, 529, 487, 514, 518, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 524, 477, 546, 525, 530, 468, 484, 492, 4, 0, 0, 0, 0, 0, 0, 0, 473, 504, 493, 499, 511, 502, 506, 516, 13, 0, 0, 0, 0, 0, 0, 0, 530, 515, 517, 538, 474, 538, 542, 474, 0, 0, 0, 0, 0, 0, 0, 0, 514, 542, 517, 528, 514, 518, 514, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4074, 4106, 4109, 4050, 4017, 4128, 4168] avg=4096.0000 max=4168 max_rank=7 min=4017 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[507, 533, 512, 533, 510, 495, 517, 505, 477, 467, 545, 488, 521, 508, 524, 507, 495, 514, 511, 529, 496, 493, 517, 484, 521, 508, 498, 497, 533, 550, 484, 564, 503, 502, 520, 528, 516, 530, 510, 540, 498, 484, 524, 549, 520, 501, 496, 499, 514, 529, 518, 523, 492, 482, 470, 509, 528, 498, 535, 508, 522, 536, 516, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[573, 479, 509, 525, 532, 478, 543, 513, 496, 498, 543, 522, 520, 491, 510, 501, 534, 498, 506, 524, 491, 535, 507, 497, 540, 545, 522, 481, 510, 488, 465, 516, 556, 512, 492, 528, 497, 538, 514, 517, 510, 539, 470, 504, 524, 517, 512, 524, 509, 491, 525, 489, 534, 537, 497, 501, 527, 517, 508, 478, 525, 499, 490, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[477, 562, 495, 474, 509, 519, 497, 501, 509, 472, 535, 513, 501, 546, 523, 506, 460, 536, 526, 519, 513, 501, 514, 541, 517, 445, 504, 505, 545, 470, 535, 523, 537, 511, 514, 496, 508, 519, 478, 482, 553, 492, 527, 518, 547, 527, 514, 503, 517, 496, 539, 481, 500, 492, 524, 514, 560, 544, 525, 498, 524, 509, 515, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[519, 527, 522, 547, 505, 537, 456, 518, 516, 533, 505, 490, 503, 504, 519, 528, 529, 489, 503, 512, 537, 492, 525, 493, 524, 522, 506, 503, 500, 540, 504, 541, 484, 476, 544, 546, 520, 529, 532, 506, 501, 516, 528, 475, 508, 547, 497, 521, 497, 519, 485, 533, 507, 503, 499, 511, 534, 489, 538, 494, 467, 522, 497, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[492, 504, 520, 487, 511, 514, 515, 515, 521, 559, 499, 527, 503, 499, 512, 499, 474, 516, 501, 531, 491, 522, 535, 547, 531, 497, 507, 523, 504, 489, 535, 505, 541, 491, 518, 533, 495, 484, 526, 546, 548, 521, 495, 476, 489, 515, 511, 496, 539, 552, 523, 534, 478, 502, 481, 491, 522, 545, 478, 495, 525, 540, 507, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[527, 519, 517, 482, 489, 506, 517, 552, 518, 493, 525, 513, 519, 497, 493, 518, 483, 503, 515, 521, 489, 504, 505, 474, 527, 497, 525, 547, 525, 517, 523, 515, 515, 536, 521, 510, 511, 537, 476, 522, 517, 524, 543, 491, 479, 518, 523, 514, 493, 517, 543, 516, 472, 528, 525, 519, 504, 475, 514, 489, 501, 522, 533, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[491, 500, 502, 498, 481, 526, 486, 543, 517, 461, 533, 538, 520, 476, 525, 537, 535, 502, 493, 523, 509, 510, 534, 508, 478, 526, 492, 471, 527, 512, 512, 520, 513, 522, 502, 498, 529, 524, 487, 507, 509, 529, 535, 512, 531, 496, 516, 492, 479, 536, 493, 533, 497, 494, 524, 521, 526, 519, 525, 533, 510, 535, 517, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[523, 537, 483, 480, 506, 505, 557, 497, 505, 509, 528, 504, 516, 550, 527, 501, 483, 465, 522, 530, 519, 501, 522, 503, 522, 527, 553, 509, 539, 497, 500, 543, 495, 517, 487, 476, 497, 542, 522, 497, 484, 511, 510, 515, 472, 540, 523, 516, 496, 514, 555, 521, 534, 527, 480, 500, 510, 503, 506, 537, 522, 500, 486, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[573, 479, 509, 525, 532, 478, 543, 513, 4, 2, 0, 0, 0, 0, 0, 0, 496, 498, 543, 522, 520, 491, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0, 534, 498, 506, 524, 491, 535, 507, 497, 16, 5, 0, 0, 0, 0, 0, 0, 540, 545, 522, 481, 510, 488, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 556, 512, 492, 528, 497, 527, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 510, 539, 470, 504, 524, 515, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 525, 489, 534, 537, 497, 501, 7, 6, 0, 0, 0, 0, 0, 0, 516, 517, 508, 478, 525, 499, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4081, 4113, 4051, 4143, 4098, 4096, 4028] avg=4096.0000 max=4158 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 533, 512, 533, 510, 495, 517, 505, 4, 2, 0, 0, 0, 0, 0, 0, 477, 467, 545, 488, 521, 508, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 495, 514, 511, 529, 496, 493, 517, 484, 17, 5, 0, 0, 0, 0, 0, 0, 521, 508, 498, 497, 533, 550, 484, 547, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 520, 528, 516, 519, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 498, 484, 524, 549, 520, 499, 496, 499, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 518, 523, 492, 482, 470, 509, 7, 6, 0, 0, 0, 0, 0, 0, 517, 498, 535, 508, 522, 536, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4037, 4061, 4138, 4138, 4069, 4050, 4157] avg=4096.0000 max=4157 max_rank=7 min=4037 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 527, 522, 547, 505, 537, 456, 518, 4, 3, 0, 0, 0, 0, 0, 0, 516, 533, 505, 490, 503, 504, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 529, 489, 503, 512, 537, 492, 525, 493, 17, 5, 0, 0, 0, 0, 0, 0, 524, 522, 506, 503, 500, 540, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 484, 476, 544, 546, 520, 518, 532, 506, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 528, 475, 508, 544, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 497, 519, 485, 533, 507, 503, 499, 511, 7, 6, 0, 0, 0, 0, 0, 0, 523, 489, 538, 494, 467, 522, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4098, 4102, 4123, 4126, 4090, 4067, 4024] avg=4096.0000 max=4138 max_rank=0 min=4024 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[477, 562, 495, 474, 509, 519, 497, 501, 4, 2, 0, 0, 0, 0, 0, 0, 509, 472, 535, 513, 501, 546, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 460, 536, 526, 519, 513, 501, 514, 541, 16, 5, 0, 0, 0, 0, 0, 0, 517, 445, 504, 505, 545, 470, 535, 507, 0, 0, 0, 0, 0, 0, 0, 0, 537, 511, 514, 496, 508, 508, 478, 482, 0, 0, 0, 0, 0, 0, 0, 0, 553, 492, 527, 518, 547, 525, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 539, 481, 500, 492, 524, 514, 7, 7, 0, 0, 0, 0, 0, 0, 548, 544, 525, 498, 524, 509, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4105, 4131, 4028, 4034, 4179, 4077, 4174] avg=4096.0000 max=4179 max_rank=5 min=4028 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 502, 498, 481, 526, 486, 543, 4, 2, 0, 0, 0, 0, 0, 0, 517, 461, 533, 538, 520, 476, 525, 537, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 493, 523, 509, 510, 534, 508, 16, 5, 0, 0, 0, 0, 0, 0, 478, 526, 492, 471, 527, 512, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 502, 498, 529, 513, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 535, 512, 531, 494, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 479, 536, 493, 533, 497, 494, 524, 521, 7, 6, 0, 0, 0, 0, 0, 0, 515, 519, 525, 533, 510, 535, 517, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4107, 4135, 4022, 4071, 4118, 4090, 4192] avg=4096.0000 max=4192 max_rank=7 min=4022 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 504, 520, 487, 511, 514, 515, 515, 3, 2, 0, 0, 0, 0, 0, 0, 521, 559, 499, 527, 503, 499, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 474, 516, 501, 531, 491, 522, 535, 547, 15, 5, 0, 0, 0, 0, 0, 0, 531, 497, 507, 523, 504, 489, 535, 490, 0, 0, 0, 0, 0, 0, 0, 0, 541, 491, 518, 533, 495, 475, 526, 546, 0, 0, 0, 0, 0, 0, 0, 0, 548, 521, 495, 476, 489, 513, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 539, 552, 523, 534, 478, 502, 481, 491, 6, 6, 0, 0, 0, 0, 0, 0, 511, 545, 478, 495, 525, 540, 507, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4119, 4137, 4076, 4125, 4049, 4112, 4087] avg=4096.0000 max=4137 max_rank=2 min=4049 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[527, 519, 517, 482, 489, 506, 517, 552, 7, 8, 0, 0, 0, 0, 0, 0, 518, 493, 525, 513, 519, 497, 493, 518, 5, 0, 0, 0, 0, 0, 0, 0, 483, 503, 515, 521, 489, 504, 505, 474, 19, 9, 0, 0, 0, 0, 0, 0, 527, 497, 525, 547, 525, 517, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 521, 510, 511, 518, 476, 522, 0, 0, 0, 0, 0, 0, 0, 0, 517, 524, 543, 491, 479, 505, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 543, 516, 472, 528, 525, 519, 12, 11, 0, 0, 0, 0, 0, 0, 484, 475, 514, 489, 501, 522, 533, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4081, 4022, 4157, 4109, 4096, 4136, 4043] avg=4096.0000 max=4157 max_rank=3 min=4022 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[523, 537, 483, 480, 506, 505, 557, 497, 4, 2, 0, 0, 0, 0, 0, 0, 505, 509, 528, 504, 516, 550, 527, 501, 0, 0, 0, 0, 0, 0, 0, 0, 483, 465, 522, 530, 519, 501, 522, 503, 17, 5, 0, 0, 0, 0, 0, 0, 522, 527, 553, 509, 539, 497, 500, 526, 0, 0, 0, 0, 0, 0, 0, 0, 495, 517, 487, 476, 497, 531, 522, 497, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 510, 515, 472, 538, 523, 516, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 555, 521, 534, 527, 480, 500, 7, 6, 0, 0, 0, 0, 0, 0, 499, 503, 506, 537, 522, 500, 486, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4140, 4067, 4173, 4022, 4069, 4140, 4063] avg=4096.0000 max=4173 max_rank=3 min=4022 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[515, 514, 530, 481, 541, 530, 528, 525, 527, 509, 499, 497, 546, 489, 481, 527, 508, 500, 481, 512, 521, 489, 498, 490, 494, 477, 536, 516, 507, 532, 462, 523, 498, 534, 506, 539, 470, 482, 507, 494, 511, 545, 531, 509, 483, 531, 480, 498, 547, 487, 525, 529, 503, 508, 542, 505, 523, 529, 526, 500, 535, 533, 520, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[493, 520, 542, 510, 495, 524, 469, 497, 495, 537, 506, 493, 503, 526, 512, 493, 508, 470, 539, 529, 526, 531, 536, 494, 526, 519, 494, 497, 540, 502, 543, 503, 508, 512, 527, 521, 541, 499, 526, 509, 540, 524, 521, 525, 454, 472, 506, 485, 520, 524, 510, 518, 501, 511, 531, 521, 509, 478, 545, 503, 501, 522, 499, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[506, 494, 492, 518, 540, 537, 488, 541, 515, 524, 524, 486, 504, 514, 512, 523, 544, 513, 532, 478, 495, 522, 503, 501, 514, 507, 537, 526, 496, 514, 499, 513, 509, 520, 545, 513, 544, 519, 511, 510, 468, 543, 505, 484, 505, 496, 488, 514, 517, 495, 513, 503, 522, 483, 513, 503, 522, 491, 485, 519, 538, 519, 541, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[531, 505, 502, 501, 481, 512, 493, 514, 479, 489, 489, 507, 501, 551, 521, 533, 520, 483, 491, 465, 516, 533, 522, 503, 506, 510, 527, 532, 527, 512, 527, 503, 534, 529, 520, 472, 479, 512, 511, 494, 533, 477, 521, 520, 517, 473, 545, 522, 503, 516, 567, 518, 529, 536, 516, 505, 512, 481, 536, 499, 550, 533, 504, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[552, 521, 540, 552, 506, 498, 512, 497, 537, 472, 518, 487, 497, 526, 522, 519, 527, 524, 501, 524, 516, 501, 515, 513, 523, 470, 484, 532, 516, 524, 546, 545, 502, 511, 498, 519, 519, 494, 518, 492, 505, 493, 531, 531, 490, 503, 520, 499, 516, 500, 528, 527, 512, 537, 447, 484, 524, 520, 498, 470, 483, 572, 501, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[512, 523, 527, 498, 527, 505, 515, 502, 495, 509, 511, 514, 499, 516, 532, 526, 537, 498, 512, 493, 527, 478, 485, 508, 487, 539, 526, 477, 487, 554, 520, 478, 536, 511, 467, 468, 525, 503, 517, 510, 528, 522, 513, 544, 469, 545, 523, 532, 490, 557, 544, 522, 503, 528, 518, 503, 530, 500, 521, 479, 487, 551, 506, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[531, 510, 527, 502, 527, 472, 528, 519, 528, 541, 518, 523, 512, 491, 496, 473, 539, 498, 478, 562, 494, 508, 499, 522, 477, 540, 485, 520, 522, 501, 515, 494, 525, 525, 488, 493, 495, 507, 512, 523, 507, 504, 528, 507, 519, 501, 531, 513, 538, 542, 520, 524, 493, 485, 525, 527, 481, 496, 535, 538, 521, 488, 504, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[482, 498, 513, 516, 524, 491, 510, 507, 474, 542, 519, 502, 525, 515, 501, 547, 514, 499, 540, 482, 522, 528, 496, 501, 491, 528, 535, 507, 524, 486, 508, 518, 521, 545, 513, 517, 547, 533, 519, 514, 481, 497, 523, 527, 504, 505, 499, 487, 505, 456, 530, 503, 514, 562, 509, 497, 507, 518, 541, 514, 472, 505, 529, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 537, 510, 495, 524, 469, 497, 0, 0, 0, 0, 0, 0, 0, 0, 495, 537, 506, 493, 503, 526, 512, 493, 5, 2, 0, 0, 0, 0, 0, 0, 508, 470, 539, 529, 526, 531, 536, 494, 20, 0, 0, 0, 0, 0, 0, 0, 526, 519, 494, 497, 540, 502, 543, 503, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 527, 521, 541, 499, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 540, 524, 521, 525, 454, 472, 506, 485, 16, 3, 0, 0, 0, 0, 0, 0, 520, 524, 487, 518, 501, 511, 531, 521, 0, 0, 0, 0, 0, 0, 0, 0, 509, 478, 545, 503, 501, 504, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4072, 4153, 4124, 4143, 4046, 4113, 4072] avg=4096.0000 max=4153 max_rank=2 min=4045 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[552, 521, 535, 552, 506, 498, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 537, 472, 518, 487, 497, 526, 522, 519, 5, 2, 0, 0, 0, 0, 0, 0, 527, 524, 501, 524, 516, 501, 515, 513, 21, 0, 0, 0, 0, 0, 0, 0, 523, 470, 484, 532, 516, 524, 546, 545, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 498, 519, 519, 494, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 531, 531, 490, 503, 520, 499, 18, 3, 0, 0, 0, 0, 0, 0, 516, 500, 504, 527, 512, 537, 447, 484, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 498, 470, 483, 552, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4085, 4142, 4140, 4053, 4093, 4027, 4055] avg=4096.0000 max=4173 max_rank=0 min=4027 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[506, 494, 487, 518, 540, 537, 488, 541, 0, 0, 0, 0, 0, 0, 0, 0, 515, 524, 524, 486, 504, 514, 512, 523, 5, 2, 0, 0, 0, 0, 0, 0, 544, 513, 532, 478, 495, 522, 503, 501, 21, 0, 0, 0, 0, 0, 0, 0, 514, 507, 537, 526, 496, 514, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 545, 513, 544, 519, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 468, 543, 505, 484, 505, 496, 488, 514, 16, 3, 0, 0, 0, 0, 0, 0, 517, 495, 489, 503, 522, 483, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 522, 491, 485, 519, 538, 501, 541, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4109, 4109, 4106, 4171, 4022, 4025, 4115] avg=4096.0000 max=4171 max_rank=4 min=4022 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 514, 525, 481, 541, 530, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 499, 497, 546, 489, 481, 527, 5, 2, 0, 0, 0, 0, 0, 0, 508, 500, 481, 512, 521, 489, 498, 490, 21, 0, 0, 0, 0, 0, 0, 0, 494, 477, 536, 516, 507, 532, 462, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 534, 506, 539, 470, 482, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 511, 545, 531, 509, 483, 531, 480, 498, 17, 3, 0, 0, 0, 0, 0, 0, 547, 487, 501, 529, 503, 508, 542, 505, 0, 0, 0, 0, 0, 0, 0, 0, 523, 529, 526, 500, 535, 514, 520, 553, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4082, 4020, 4047, 4030, 4108, 4122, 4200] avg=4096.0000 max=4200 max_rank=7 min=4020 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[531, 505, 497, 501, 481, 512, 493, 514, 0, 0, 0, 0, 0, 0, 0, 0, 479, 489, 489, 507, 501, 551, 521, 533, 5, 2, 0, 0, 0, 0, 0, 0, 520, 483, 491, 465, 516, 533, 522, 503, 23, 0, 0, 0, 0, 0, 0, 0, 506, 510, 527, 532, 527, 512, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 534, 529, 520, 472, 479, 512, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 533, 477, 521, 520, 517, 473, 545, 522, 17, 3, 0, 0, 0, 0, 0, 0, 503, 516, 541, 518, 529, 536, 516, 505, 0, 0, 0, 0, 0, 0, 0, 0, 512, 481, 536, 499, 550, 514, 504, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4077, 4056, 4144, 4051, 4128, 4164, 4114] avg=4096.0000 max=4164 max_rank=6 min=4034 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 523, 517, 498, 527, 505, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 511, 514, 499, 516, 532, 526, 10, 4, 6, 0, 0, 0, 0, 0, 537, 498, 512, 493, 527, 478, 485, 508, 26, 0, 0, 0, 0, 0, 0, 0, 487, 539, 526, 477, 487, 542, 520, 478, 0, 0, 0, 0, 0, 0, 0, 0, 536, 511, 467, 468, 525, 503, 517, 510, 6, 0, 0, 0, 0, 0, 0, 0, 528, 522, 513, 544, 469, 545, 523, 532, 21, 4, 0, 0, 0, 0, 0, 0, 490, 557, 514, 522, 503, 528, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 521, 479, 487, 526, 506, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4122, 4064, 4056, 4043, 4201, 4135, 4048] avg=4096.0000 max=4201 max_rank=5 min=4043 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[531, 510, 522, 502, 527, 472, 528, 519, 0, 0, 0, 0, 0, 0, 0, 0, 528, 541, 518, 523, 512, 491, 496, 473, 5, 2, 0, 0, 0, 0, 0, 0, 539, 498, 478, 562, 494, 508, 499, 522, 21, 0, 0, 0, 0, 0, 0, 0, 477, 540, 485, 520, 522, 501, 515, 494, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 488, 493, 495, 507, 512, 523, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 528, 507, 519, 501, 531, 513, 15, 3, 0, 0, 0, 0, 0, 0, 538, 542, 496, 524, 493, 485, 525, 527, 0, 0, 0, 0, 0, 0, 0, 0, 481, 496, 535, 538, 521, 471, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4089, 4121, 4054, 4068, 4128, 4130, 4067] avg=4096.0000 max=4130 max_rank=6 min=4054 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[482, 498, 508, 516, 524, 491, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 474, 542, 519, 502, 525, 515, 501, 547, 5, 2, 0, 0, 0, 0, 0, 0, 514, 499, 540, 482, 522, 528, 496, 501, 21, 0, 0, 0, 0, 0, 0, 0, 491, 528, 535, 507, 524, 486, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 521, 545, 513, 517, 547, 533, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 481, 497, 523, 527, 504, 505, 499, 487, 16, 3, 0, 0, 0, 0, 0, 0, 505, 456, 506, 503, 514, 562, 509, 497, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 541, 514, 472, 487, 529, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4132, 4103, 4097, 4209, 4042, 4052, 4097] avg=4096.0000 max=4209 max_rank=4 min=4036 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[525, 508, 546, 499, 529, 539, 497, 494, 544, 545, 536, 500, 490, 516, 521, 497, 492, 500, 544, 511, 506, 476, 520, 510, 531, 506, 513, 473, 512, 514, 509, 505, 546, 534, 508, 530, 506, 497, 517, 510, 521, 505, 474, 552, 522, 534, 526, 501, 468, 497, 499, 530, 474, 521, 493, 506, 514, 522, 532, 553, 529, 503, 468, 468] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[507, 508, 519, 506, 512, 489, 507, 500, 508, 471, 518, 509, 512, 483, 499, 509, 477, 479, 554, 525, 520, 521, 551, 507, 527, 549, 497, 467, 548, 518, 499, 523, 506, 531, 473, 532, 498, 502, 510, 489, 503, 521, 521, 478, 522, 535, 491, 506, 472, 510, 505, 525, 527, 531, 519, 524, 468, 510, 542, 559, 548, 551, 514, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[493, 501, 536, 527, 516, 505, 490, 490, 492, 537, 518, 494, 517, 505, 513, 511, 490, 511, 521, 531, 559, 544, 490, 533, 475, 524, 521, 530, 479, 507, 526, 487, 493, 499, 525, 517, 533, 535, 501, 491, 497, 506, 506, 519, 479, 509, 525, 503, 537, 545, 504, 469, 470, 504, 563, 533, 500, 531, 509, 512, 539, 510, 511, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[525, 485, 514, 537, 552, 476, 544, 523, 508, 515, 504, 518, 518, 523, 503, 512, 505, 511, 519, 488, 517, 449, 520, 509, 510, 491, 487, 547, 527, 503, 527, 484, 484, 548, 518, 503, 523, 519, 560, 520, 476, 509, 516, 491, 498, 517, 529, 516, 525, 510, 524, 469, 511, 516, 489, 512, 483, 522, 525, 532, 526, 528, 496, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[508, 557, 533, 591, 502, 525, 511, 528, 520, 480, 547, 472, 518, 515, 492, 550, 495, 528, 510, 560, 502, 537, 500, 519, 531, 506, 480, 490, 513, 511, 515, 528, 521, 474, 537, 523, 520, 524, 504, 491, 472, 502, 500, 516, 507, 517, 513, 543, 508, 476, 476, 483, 486, 490, 507, 476, 535, 477, 495, 528, 532, 520, 543, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[511, 525, 531, 540, 487, 527, 519, 556, 553, 525, 476, 484, 501, 500, 472, 468, 511, 527, 537, 529, 481, 508, 488, 503, 499, 547, 539, 517, 520, 524, 503, 481, 521, 494, 502, 522, 522, 477, 482, 498, 479, 535, 487, 550, 522, 521, 490, 480, 501, 546, 524, 530, 504, 481, 524, 521, 516, 542, 555, 493, 502, 498, 525, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[493, 506, 511, 505, 528, 522, 534, 510, 504, 486, 532, 484, 539, 500, 511, 510, 515, 522, 513, 497, 538, 494, 518, 492, 529, 524, 516, 566, 499, 487, 505, 484, 525, 517, 524, 540, 571, 514, 533, 512, 535, 522, 503, 454, 536, 510, 513, 486, 536, 481, 500, 485, 470, 557, 553, 483, 532, 514, 521, 496, 497, 511, 485, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[539, 522, 549, 490, 515, 464, 520, 495, 469, 513, 506, 518, 488, 543, 507, 549, 517, 506, 486, 496, 504, 508, 491, 514, 534, 505, 539, 536, 522, 549, 497, 516, 513, 519, 483, 546, 497, 458, 552, 525, 534, 518, 528, 505, 517, 526, 500, 509, 504, 512, 508, 511, 521, 563, 507, 503, 514, 514, 469, 519, 498, 474, 528, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 508, 485, 506, 512, 489, 507, 500, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 518, 509, 512, 483, 499, 509, 18, 3, 3, 0, 0, 0, 0, 0, 477, 479, 554, 525, 520, 521, 551, 507, 4, 0, 0, 0, 0, 0, 0, 0, 527, 542, 497, 467, 548, 518, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 506, 531, 473, 517, 498, 502, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 521, 478, 522, 535, 491, 506, 12, 3, 0, 0, 0, 0, 0, 0, 472, 510, 505, 525, 527, 531, 519, 524, 31, 0, 0, 0, 0, 0, 0, 0, 468, 510, 542, 541, 548, 551, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4033, 4138, 4121, 4026, 4092, 4144, 4200] avg=4096.0000 max=4200 max_rank=7 min=4014 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[525, 485, 480, 537, 552, 476, 544, 523, 0, 0, 0, 0, 0, 0, 0, 0, 508, 515, 504, 518, 518, 523, 503, 512, 17, 3, 3, 0, 0, 0, 0, 0, 505, 511, 519, 488, 517, 449, 520, 509, 3, 0, 0, 0, 0, 0, 0, 0, 510, 485, 487, 547, 527, 503, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 484, 548, 518, 488, 523, 519, 560, 520, 0, 0, 0, 0, 0, 0, 0, 0, 476, 509, 516, 491, 498, 517, 529, 516, 12, 3, 0, 0, 0, 0, 0, 0, 525, 510, 524, 469, 511, 516, 489, 512, 31, 0, 0, 0, 0, 0, 0, 0, 483, 522, 525, 515, 526, 528, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4124, 4021, 4070, 4160, 4067, 4087, 4117] avg=4096.0000 max=4160 max_rank=4 min=4021 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[525, 508, 510, 499, 529, 539, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0, 544, 545, 536, 500, 490, 516, 521, 497, 17, 3, 3, 0, 0, 0, 0, 0, 492, 500, 544, 511, 506, 476, 520, 510, 3, 0, 0, 0, 0, 0, 0, 0, 531, 500, 513, 473, 512, 514, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 546, 534, 508, 515, 506, 497, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 474, 552, 522, 534, 526, 501, 12, 3, 0, 0, 0, 0, 0, 0, 468, 497, 499, 530, 474, 521, 493, 506, 33, 0, 0, 0, 0, 0, 0, 0, 514, 522, 532, 536, 529, 503, 468, 468, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4172, 4062, 4057, 4133, 4150, 4021, 4072] avg=4096.0000 max=4172 max_rank=1 min=4021 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 501, 501, 527, 516, 505, 490, 490, 0, 0, 0, 0, 0, 0, 0, 0, 492, 537, 518, 494, 517, 505, 513, 511, 16, 3, 3, 0, 0, 0, 0, 0, 490, 511, 521, 531, 559, 544, 490, 533, 4, 0, 0, 0, 0, 0, 0, 0, 475, 517, 521, 530, 479, 507, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 493, 499, 525, 502, 533, 535, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 497, 506, 506, 519, 479, 509, 525, 503, 12, 3, 0, 0, 0, 0, 0, 0, 537, 545, 504, 469, 470, 504, 563, 533, 32, 0, 0, 0, 0, 0, 0, 0, 500, 531, 509, 496, 539, 510, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4109, 4183, 4042, 4079, 4059, 4157, 4116] avg=4096.0000 max=4183 max_rank=2 min=4023 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 557, 498, 591, 502, 525, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 520, 480, 547, 472, 518, 515, 492, 550, 17, 3, 3, 0, 0, 0, 0, 0, 495, 528, 510, 560, 502, 537, 500, 519, 3, 0, 0, 0, 0, 0, 0, 0, 531, 500, 480, 490, 513, 511, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 521, 474, 537, 508, 520, 524, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 472, 502, 500, 516, 507, 517, 513, 543, 12, 3, 0, 0, 0, 0, 0, 0, 508, 476, 476, 483, 486, 490, 507, 476, 32, 0, 0, 0, 0, 0, 0, 0, 535, 477, 495, 511, 532, 520, 543, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4220, 4117, 4154, 4068, 4079, 4085, 3934, 4111] avg=4096.0000 max=4220 max_rank=0 min=3934 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 506, 477, 505, 528, 522, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 504, 486, 532, 484, 539, 500, 511, 510, 15, 3, 3, 0, 0, 0, 0, 0, 515, 522, 513, 497, 538, 494, 518, 492, 4, 0, 0, 0, 0, 0, 0, 0, 529, 517, 516, 566, 499, 487, 505, 484, 0, 0, 0, 0, 0, 0, 0, 0, 525, 517, 524, 524, 571, 514, 533, 512, 0, 0, 0, 0, 0, 0, 0, 0, 535, 522, 503, 454, 536, 510, 513, 486, 13, 3, 0, 0, 0, 0, 0, 0, 536, 481, 500, 485, 470, 557, 553, 483, 31, 0, 0, 0, 0, 0, 0, 0, 532, 514, 521, 481, 497, 511, 485, 478, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4087, 4093, 4103, 4220, 4075, 4096, 4019] avg=4096.0000 max=4220 max_rank=4 min=4019 min_rank=7 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 525, 490, 540, 487, 527, 519, 556, 0, 0, 0, 0, 0, 0, 0, 0, 553, 525, 476, 484, 501, 500, 472, 468, 19, 8, 5, 0, 0, 0, 0, 0, 511, 527, 537, 529, 481, 508, 488, 503, 8, 0, 0, 0, 0, 0, 0, 0, 499, 532, 539, 517, 520, 524, 503, 481, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 502, 498, 522, 477, 482, 498, 0, 0, 0, 0, 0, 0, 0, 0, 479, 535, 487, 550, 522, 521, 490, 480, 16, 7, 0, 0, 0, 0, 0, 0, 501, 546, 524, 530, 504, 481, 524, 521, 36, 0, 0, 0, 0, 0, 0, 0, 516, 542, 555, 474, 502, 498, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4011, 4092, 4115, 3994, 4087, 4167, 4147] avg=4096.0000 max=4167 max_rank=6 min=3994 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[539, 522, 513, 490, 515, 464, 520, 495, 0, 0, 0, 0, 0, 0, 0, 0, 469, 513, 506, 518, 488, 543, 507, 549, 16, 3, 3, 0, 0, 0, 0, 0, 517, 506, 486, 496, 504, 508, 491, 514, 3, 0, 0, 0, 0, 0, 0, 0, 534, 499, 539, 536, 522, 549, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 483, 530, 497, 458, 552, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 518, 528, 505, 517, 526, 500, 509, 13, 3, 0, 0, 0, 0, 0, 0, 504, 512, 508, 511, 521, 563, 507, 503, 33, 0, 0, 0, 0, 0, 0, 0, 514, 514, 469, 503, 498, 474, 528, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4115, 4025, 4192, 4077, 4153, 4162, 3986] avg=4096.0000 max=4192 max_rank=3 min=3986 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[525, 514, 494, 476, 502, 527, 488, 521, 516, 512, 507, 499, 511, 522, 478, 508, 534, 531, 479, 580, 529, 475, 516, 507, 516, 502, 541, 521, 502, 486, 518, 497, 490, 510, 529, 559, 510, 517, 494, 503, 569, 498, 539, 510, 520, 501, 494, 473, 542, 519, 518, 508, 520, 524, 496, 499, 509, 493, 503, 535, 491, 512, 509, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[505, 523, 511, 503, 525, 518, 496, 536, 532, 496, 471, 494, 518, 519, 539, 531, 565, 480, 501, 504, 518, 509, 514, 507, 520, 509, 498, 484, 519, 535, 515, 532, 531, 506, 532, 505, 520, 537, 480, 500, 512, 546, 531, 514, 482, 518, 474, 512, 508, 514, 496, 537, 496, 509, 509, 526, 522, 489, 486, 508, 515, 488, 523, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[491, 503, 535, 505, 479, 507, 483, 489, 518, 516, 519, 519, 540, 511, 528, 493, 507, 499, 534, 482, 485, 506, 553, 517, 497, 527, 529, 515, 508, 505, 517, 518, 512, 487, 542, 526, 502, 502, 493, 503, 523, 541, 523, 533, 523, 520, 520, 463, 502, 480, 498, 489, 554, 509, 480, 520, 550, 517, 502, 509, 552, 556, 497, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[573, 515, 512, 525, 521, 524, 518, 530, 519, 492, 481, 481, 540, 515, 534, 537, 504, 513, 515, 501, 503, 507, 478, 518, 523, 476, 492, 509, 528, 493, 529, 496, 537, 482, 507, 544, 545, 525, 510, 520, 533, 497, 523, 528, 534, 513, 500, 491, 462, 510, 538, 507, 488, 481, 507, 488, 513, 536, 513, 498, 480, 507, 552, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[506, 514, 485, 523, 513, 510, 526, 500, 529, 514, 537, 532, 481, 518, 554, 520, 517, 498, 515, 464, 505, 465, 529, 521, 514, 520, 523, 517, 498, 536, 504, 525, 514, 512, 521, 510, 519, 537, 540, 531, 522, 511, 532, 515, 523, 511, 520, 496, 500, 543, 498, 499, 490, 494, 513, 516, 497, 501, 499, 501, 480, 530, 473, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[490, 539, 525, 541, 547, 488, 466, 545, 527, 507, 526, 478, 485, 503, 518, 511, 542, 515, 488, 495, 513, 525, 473, 541, 500, 484, 482, 533, 530, 539, 502, 447, 482, 500, 526, 502, 513, 512, 547, 513, 532, 535, 522, 514, 493, 512, 510, 540, 505, 529, 548, 485, 507, 551, 468, 520, 507, 529, 568, 490, 518, 499, 515, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[483, 551, 489, 488, 508, 473, 497, 526, 479, 518, 532, 576, 493, 514, 513, 480, 584, 485, 538, 548, 555, 529, 535, 490, 518, 500, 493, 508, 537, 530, 515, 527, 537, 525, 490, 511, 545, 506, 499, 500, 489, 477, 482, 518, 529, 505, 510, 503, 519, 485, 516, 529, 500, 495, 506, 515, 524, 486, 493, 527, 534, 485, 524, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[518, 495, 493, 514, 535, 482, 514, 520, 528, 528, 510, 519, 496, 534, 513, 511, 538, 518, 495, 483, 490, 541, 521, 503, 510, 532, 521, 531, 506, 509, 516, 500, 529, 446, 549, 504, 538, 511, 503, 531, 533, 501, 512, 552, 506, 503, 513, 491, 514, 497, 497, 484, 504, 485, 465, 514, 526, 535, 557, 503, 555, 510, 469, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 523, 511, 503, 525, 518, 496, 536, 8, 3, 0, 0, 0, 0, 0, 0, 532, 496, 471, 494, 518, 519, 525, 531, 0, 0, 0, 0, 0, 0, 0, 0, 558, 480, 501, 504, 518, 509, 514, 507, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 498, 484, 519, 535, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 506, 508, 505, 520, 537, 480, 500, 0, 0, 0, 0, 0, 0, 0, 0, 500, 546, 531, 514, 482, 518, 474, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 496, 537, 496, 509, 509, 526, 24, 14, 4, 0, 0, 0, 0, 0, 522, 489, 486, 508, 515, 488, 523, 515, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4086, 4091, 4112, 4087, 4077, 4137, 4050] avg=4096.0000 max=4137 max_rank=6 min=4050 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[490, 539, 525, 541, 547, 488, 466, 545, 8, 3, 0, 0, 0, 0, 0, 0, 527, 507, 526, 478, 485, 503, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 515, 488, 495, 513, 525, 473, 541, 0, 0, 0, 0, 0, 0, 0, 0, 500, 484, 482, 533, 530, 539, 502, 447, 0, 0, 0, 0, 0, 0, 0, 0, 482, 500, 502, 502, 513, 512, 547, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 535, 522, 514, 493, 512, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 548, 485, 507, 551, 468, 520, 24, 13, 4, 0, 0, 0, 0, 0, 507, 529, 568, 490, 518, 499, 515, 471, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4042, 4085, 4017, 4071, 4146, 4154, 4101] avg=4096.0000 max=4154 max_rank=6 min=4017 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[525, 514, 494, 476, 502, 527, 488, 521, 8, 2, 0, 0, 0, 0, 0, 0, 516, 512, 507, 499, 511, 522, 466, 508, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 479, 580, 529, 475, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 541, 521, 502, 486, 518, 497, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 505, 559, 510, 517, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 557, 498, 539, 510, 520, 501, 494, 473, 0, 0, 0, 0, 0, 0, 0, 0, 542, 519, 518, 508, 520, 524, 496, 499, 24, 12, 4, 0, 0, 0, 0, 0, 509, 493, 503, 535, 491, 512, 509, 540, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4041, 4145, 4083, 4088, 4092, 4166, 4096] avg=4096.0000 max=4166 max_rank=6 min=4041 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[573, 515, 512, 525, 521, 524, 518, 530, 8, 2, 0, 0, 0, 0, 0, 0, 519, 492, 481, 481, 540, 515, 520, 537, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 515, 501, 503, 507, 478, 518, 0, 0, 0, 0, 0, 0, 0, 0, 523, 476, 492, 509, 528, 493, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0, 537, 482, 484, 544, 545, 525, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 521, 497, 523, 528, 534, 513, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 462, 510, 538, 507, 488, 481, 507, 488, 23, 14, 4, 0, 0, 0, 0, 0, 513, 536, 513, 498, 480, 507, 552, 497, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4228, 4085, 4034, 4046, 4147, 4107, 4022, 4099] avg=4096.0000 max=4228 max_rank=0 min=4022 min_rank=6 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[491, 503, 535, 505, 479, 507, 483, 489, 8, 2, 0, 0, 0, 0, 0, 0, 518, 516, 519, 519, 540, 511, 514, 493, 0, 0, 0, 0, 0, 0, 0, 0, 501, 499, 534, 482, 485, 506, 553, 517, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 529, 515, 508, 505, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0, 512, 487, 517, 526, 502, 502, 493, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 523, 533, 523, 520, 520, 463, 0, 0, 0, 0, 0, 0, 0, 0, 502, 480, 498, 489, 554, 509, 480, 520, 25, 14, 4, 0, 0, 0, 0, 0, 550, 517, 502, 509, 552, 556, 497, 505, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4130, 4077, 4116, 4042, 4134, 4075, 4192] avg=4096.0000 max=4192 max_rank=7 min=4002 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[506, 514, 485, 523, 513, 510, 526, 500, 11, 6, 0, 0, 0, 0, 0, 0, 529, 514, 537, 532, 481, 518, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 500, 498, 515, 464, 505, 465, 529, 521, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 523, 517, 498, 536, 504, 525, 4, 0, 0, 0, 0, 0, 0, 0, 514, 512, 493, 510, 519, 537, 540, 531, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 532, 515, 523, 511, 520, 496, 0, 0, 0, 0, 0, 0, 0, 0, 500, 543, 498, 499, 490, 494, 513, 516, 28, 19, 8, 0, 0, 0, 0, 0, 497, 501, 499, 501, 480, 530, 473, 507, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4166, 3997, 4141, 4156, 4111, 4108, 3995] avg=4096.0000 max=4166 max_rank=1 min=3995 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[483, 551, 489, 488, 508, 473, 497, 526, 7, 3, 0, 0, 0, 0, 0, 0, 479, 518, 532, 576, 493, 514, 500, 480, 0, 0, 0, 0, 0, 0, 0, 0, 577, 485, 538, 548, 555, 529, 535, 490, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 493, 508, 537, 530, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 468, 511, 545, 506, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 478, 477, 482, 518, 529, 505, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 519, 485, 516, 529, 500, 495, 506, 515, 22, 13, 4, 0, 0, 0, 0, 0, 524, 486, 493, 527, 534, 485, 524, 492, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4092, 4257, 4128, 4091, 4002, 4104, 4069] avg=4096.0000 max=4257 max_rank=2 min=4002 min_rank=5 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 495, 493, 514, 535, 482, 514, 520, 8, 3, 0, 0, 0, 0, 0, 0, 528, 528, 510, 519, 496, 534, 500, 511, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 495, 483, 490, 541, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 521, 531, 506, 509, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 529, 446, 524, 504, 538, 511, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 521, 501, 512, 552, 506, 503, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 497, 484, 504, 485, 465, 514, 25, 13, 4, 0, 0, 0, 0, 0, 526, 535, 557, 503, 555, 510, 469, 507, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4126, 4082, 4125, 4086, 4099, 4002, 4166] avg=4096.0000 max=4166 max_rank=7 min=4002 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[542, 489, 535, 515, 485, 490, 484, 473, 544, 505, 489, 497, 503, 480, 535, 517, 534, 508, 516, 491, 515, 536, 508, 527, 531, 487, 530, 499, 500, 547, 503, 533, 566, 475, 478, 539, 526, 497, 515, 499, 506, 502, 507, 503, 499, 500, 515, 493, 534, 530, 532, 465, 530, 524, 509, 504, 527, 542, 528, 512, 505, 526, 495, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[538, 486, 489, 539, 493, 547, 532, 508, 518, 524, 515, 540, 531, 530, 528, 497, 529, 496, 522, 503, 480, 475, 524, 485, 526, 527, 487, 539, 491, 470, 539, 518, 523, 497, 508, 506, 520, 486, 495, 496, 542, 485, 504, 467, 494, 495, 536, 495, 564, 529, 548, 527, 500, 548, 503, 513, 494, 474, 489, 547, 510, 541, 505, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[537, 540, 508, 488, 521, 501, 492, 512, 512, 507, 535, 533, 525, 476, 525, 517, 496, 514, 521, 505, 472, 515, 510, 508, 546, 505, 495, 530, 486, 520, 510, 528, 518, 513, 520, 532, 501, 509, 523, 491, 494, 539, 505, 532, 521, 514, 502, 518, 515, 511, 517, 526, 506, 500, 486, 513, 527, 486, 492, 471, 574, 512, 478, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[477, 513, 496, 518, 514, 520, 483, 511, 508, 500, 529, 531, 543, 542, 517, 504, 524, 521, 470, 530, 555, 490, 521, 461, 519, 521, 506, 509, 513, 513, 539, 475, 508, 517, 492, 507, 488, 490, 563, 542, 567, 498, 537, 516, 524, 524, 520, 531, 494, 518, 474, 524, 507, 539, 501, 512, 543, 456, 509, 504, 492, 494, 506, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[528, 504, 528, 509, 482, 520, 489, 524, 503, 567, 499, 460, 517, 498, 529, 486, 511, 510, 506, 524, 555, 545, 487, 527, 496, 511, 473, 502, 532, 506, 512, 512, 482, 501, 519, 506, 529, 497, 517, 504, 498, 527, 492, 507, 553, 544, 529, 509, 500, 514, 518, 505, 486, 531, 522, 531, 513, 528, 519, 530, 509, 517, 480, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[518, 529, 509, 461, 507, 504, 486, 469, 515, 553, 491, 524, 498, 496, 490, 490, 505, 548, 499, 500, 496, 496, 528, 482, 558, 513, 503, 514, 546, 539, 523, 521, 551, 490, 526, 526, 501, 548, 530, 515, 489, 506, 559, 543, 519, 508, 483, 490, 509, 493, 523, 506, 514, 519, 529, 510, 517, 505, 510, 485, 501, 514, 548, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[512, 535, 538, 548, 519, 519, 503, 486, 501, 475, 552, 506, 505, 539, 519, 532, 481, 498, 491, 507, 521, 506, 481, 513, 525, 474, 527, 530, 520, 543, 547, 517, 499, 489, 533, 547, 508, 528, 515, 524, 494, 465, 497, 531, 483, 532, 521, 534, 488, 518, 496, 523, 532, 495, 514, 510, 498, 498, 500, 485, 531, 500, 513, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[508, 532, 464, 515, 485, 509, 528, 518, 512, 528, 537, 567, 534, 523, 521, 487, 524, 515, 521, 501, 475, 488, 478, 513, 513, 511, 505, 516, 478, 550, 517, 534, 487, 484, 510, 515, 514, 517, 513, 484, 537, 514, 493, 503, 528, 478, 537, 553, 469, 539, 532, 496, 529, 517, 532, 522, 499, 534, 540, 514, 454, 473, 519, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[542, 489, 535, 515, 485, 490, 484, 473, 19, 6, 0, 0, 0, 0, 0, 0, 544, 505, 489, 497, 503, 480, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 516, 491, 515, 536, 508, 527, 31, 3, 0, 0, 0, 0, 0, 0, 502, 487, 530, 499, 500, 547, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 566, 475, 478, 539, 526, 497, 515, 499, 2, 0, 0, 0, 0, 0, 0, 0, 506, 502, 507, 503, 499, 500, 507, 493, 0, 0, 0, 0, 0, 0, 0, 0, 534, 530, 532, 465, 530, 505, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 527, 542, 528, 512, 505, 526, 495, 537, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4039, 4169, 4101, 4097, 4017, 4109, 4198] avg=4096.0000 max=4198 max_rank=7 min=4017 min_rank=5 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[528, 504, 528, 509, 482, 520, 489, 524, 19, 6, 0, 0, 0, 0, 0, 0, 503, 567, 499, 460, 517, 498, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 506, 524, 555, 545, 487, 527, 30, 3, 0, 0, 0, 0, 0, 0, 469, 511, 473, 502, 532, 506, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 519, 506, 529, 497, 517, 504, 2, 0, 0, 0, 0, 0, 0, 0, 498, 527, 492, 507, 553, 544, 521, 509, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 518, 505, 486, 512, 522, 531, 0, 0, 0, 0, 0, 0, 0, 0, 513, 528, 519, 530, 509, 517, 480, 499, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4029, 4198, 4017, 4057, 4151, 4088, 4119] avg=4096.0000 max=4198 max_rank=2 min=4017 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[538, 486, 489, 539, 493, 547, 532, 508, 19, 6, 0, 0, 0, 0, 0, 0, 518, 524, 515, 540, 531, 530, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 522, 503, 480, 475, 524, 485, 30, 3, 0, 0, 0, 0, 0, 0, 497, 527, 487, 539, 491, 470, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 508, 506, 520, 486, 495, 496, 2, 0, 0, 0, 0, 0, 0, 0, 542, 485, 504, 467, 494, 495, 528, 495, 0, 0, 0, 0, 0, 0, 0, 0, 564, 529, 548, 527, 500, 529, 503, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 474, 489, 547, 510, 541, 505, 501, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4153, 4047, 4068, 4033, 4010, 4213, 4087] avg=4096.0000 max=4213 max_rank=6 min=4010 min_rank=5 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[537, 540, 508, 488, 521, 501, 492, 512, 18, 6, 0, 0, 0, 0, 0, 0, 512, 507, 535, 533, 525, 476, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 521, 505, 472, 515, 510, 508, 30, 4, 0, 0, 0, 0, 0, 0, 515, 505, 495, 530, 486, 520, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 520, 532, 501, 509, 523, 491, 2, 0, 0, 0, 0, 0, 0, 0, 494, 539, 505, 532, 521, 514, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 517, 526, 506, 482, 486, 513, 0, 0, 0, 0, 0, 0, 0, 0, 527, 486, 492, 471, 574, 512, 478, 532, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4100, 4075, 4089, 4109, 4117, 4056, 4099] avg=4096.0000 max=4123 max_rank=0 min=4056 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 529, 509, 461, 507, 504, 486, 469, 18, 6, 0, 0, 0, 0, 0, 0, 515, 553, 491, 524, 498, 496, 462, 490, 0, 0, 0, 0, 0, 0, 0, 0, 505, 548, 499, 500, 496, 496, 528, 482, 28, 4, 0, 0, 0, 0, 0, 0, 527, 513, 503, 514, 546, 539, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 551, 490, 526, 526, 501, 548, 530, 515, 2, 0, 0, 0, 0, 0, 0, 0, 489, 506, 559, 543, 519, 508, 475, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 523, 506, 514, 501, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 510, 485, 501, 514, 548, 490, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4029, 4086, 4186, 4189, 4089, 4085, 4097] avg=4096.0000 max=4189 max_rank=4 min=4007 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[477, 513, 496, 518, 514, 520, 483, 511, 19, 6, 0, 0, 0, 0, 0, 0, 508, 500, 529, 531, 543, 542, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 524, 521, 470, 530, 555, 490, 521, 461, 30, 3, 0, 0, 0, 0, 0, 0, 491, 521, 506, 509, 513, 513, 539, 475, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 492, 507, 488, 490, 563, 542, 2, 0, 0, 0, 0, 0, 0, 0, 567, 498, 537, 516, 524, 524, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 474, 524, 507, 520, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 543, 456, 509, 504, 492, 494, 506, 498, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4144, 4105, 4067, 4109, 4209, 4050, 4027] avg=4096.0000 max=4209 max_rank=5 min=4027 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 535, 538, 548, 519, 519, 503, 486, 22, 12, 0, 0, 0, 0, 0, 0, 501, 475, 552, 506, 505, 539, 485, 532, 0, 0, 0, 0, 0, 0, 0, 0, 481, 498, 491, 507, 521, 506, 481, 513, 34, 8, 0, 0, 0, 0, 0, 0, 486, 474, 527, 530, 520, 543, 547, 517, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 533, 547, 508, 528, 515, 524, 5, 0, 0, 0, 0, 0, 0, 0, 494, 465, 497, 531, 483, 532, 504, 534, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 496, 523, 532, 468, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 498, 498, 500, 485, 531, 500, 513, 497, 31, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4095, 4040, 4144, 4148, 4040, 4049, 4058] avg=4096.0000 max=4194 max_rank=0 min=4040 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[508, 532, 464, 515, 485, 509, 528, 518, 18, 6, 0, 0, 0, 0, 0, 0, 512, 528, 537, 567, 534, 523, 491, 487, 0, 0, 0, 0, 0, 0, 0, 0, 524, 515, 521, 501, 475, 488, 478, 513, 30, 3, 0, 0, 0, 0, 0, 0, 485, 511, 505, 516, 478, 550, 517, 534, 0, 0, 0, 0, 0, 0, 0, 0, 487, 484, 510, 515, 514, 517, 513, 484, 2, 0, 0, 0, 0, 0, 0, 0, 537, 514, 493, 503, 528, 478, 529, 553, 0, 0, 0, 0, 0, 0, 0, 0, 469, 539, 532, 496, 529, 499, 532, 522, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 540, 514, 454, 473, 519, 525, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4179, 4048, 4096, 4026, 4135, 4118, 4083] avg=4096.0000 max=4179 max_rank=1 min=4026 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 15:43:14] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 276.0 | throughput per GPU (TFLOP/s/GPU): 447.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.488397E+00 | loss scale: 1.0 | grad norm: 3.422 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[507, 479, 530, 513, 494, 503, 496, 515, 498, 497, 532, 550, 500, 501, 514, 540, 527, 493, 477, 541, 496, 501, 509, 475, 537, 486, 511, 552, 500, 493, 514, 502, 530, 526, 525, 505, 538, 559, 560, 516, 553, 512, 481, 489, 537, 528, 556, 517, 464, 495, 491, 547, 471, 497, 522, 497, 522, 494, 528, 486, 484, 533, 522, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[516, 502, 506, 559, 564, 532, 504, 529, 505, 553, 492, 467, 498, 520, 483, 531, 524, 534, 530, 540, 521, 488, 496, 507, 505, 496, 484, 500, 513, 503, 504, 500, 535, 502, 486, 492, 495, 506, 521, 498, 524, 507, 555, 491, 542, 516, 493, 500, 481, 489, 471, 516, 502, 525, 503, 535, 504, 490, 539, 530, 498, 527, 562, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[495, 524, 499, 528, 530, 496, 509, 489, 524, 510, 514, 499, 484, 534, 517, 486, 489, 514, 556, 497, 521, 513, 531, 511, 499, 533, 510, 499, 523, 521, 512, 521, 493, 501, 521, 505, 521, 529, 518, 518, 520, 530, 533, 504, 483, 504, 525, 493, 540, 529, 498, 542, 488, 473, 516, 513, 492, 497, 540, 522, 514, 502, 515, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[499, 540, 527, 521, 523, 509, 487, 549, 503, 500, 519, 494, 500, 503, 513, 507, 524, 479, 589, 489, 478, 510, 518, 470, 526, 502, 515, 522, 530, 519, 487, 525, 509, 514, 531, 520, 454, 470, 529, 470, 499, 511, 519, 526, 505, 569, 491, 493, 531, 525, 523, 514, 534, 514, 537, 528, 543, 501, 512, 534, 499, 493, 478, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[507, 507, 517, 538, 521, 465, 523, 515, 513, 520, 509, 484, 516, 517, 534, 527, 536, 541, 520, 507, 504, 528, 514, 514, 546, 511, 489, 527, 552, 475, 510, 488, 495, 517, 512, 511, 573, 501, 544, 496, 514, 480, 510, 512, 550, 512, 503, 537, 493, 542, 525, 489, 462, 508, 498, 493, 483, 542, 487, 494, 491, 512, 508, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[480, 516, 482, 529, 509, 545, 513, 534, 512, 444, 542, 505, 510, 489, 519, 494, 467, 498, 534, 512, 485, 531, 484, 518, 554, 533, 482, 516, 561, 519, 518, 541, 526, 521, 480, 529, 504, 509, 532, 517, 512, 554, 513, 510, 510, 516, 505, 509, 528, 495, 558, 488, 508, 516, 499, 503, 526, 516, 520, 505, 504, 504, 484, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[485, 538, 502, 511, 514, 461, 506, 502, 533, 560, 504, 520, 506, 496, 506, 510, 521, 508, 446, 489, 546, 499, 501, 532, 501, 517, 495, 515, 475, 541, 509, 536, 532, 504, 498, 486, 518, 505, 533, 474, 518, 557, 560, 515, 524, 496, 533, 517, 522, 458, 493, 513, 519, 531, 507, 534, 500, 517, 539, 499, 545, 531, 519, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[546, 534, 495, 506, 525, 540, 532, 508, 490, 522, 551, 533, 489, 548, 547, 481, 518, 483, 503, 474, 510, 526, 486, 530, 519, 535, 497, 480, 560, 532, 524, 538, 499, 507, 504, 513, 508, 489, 514, 511, 505, 493, 478, 452, 533, 500, 520, 518, 532, 514, 463, 549, 544, 529, 505, 511, 520, 553, 520, 487, 507, 505, 475, 448] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 540, 527, 508, 523, 509, 487, 549, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 519, 494, 500, 503, 513, 507, 6, 0, 0, 0, 0, 0, 0, 0, 524, 479, 589, 489, 478, 510, 518, 470, 18, 0, 0, 0, 0, 0, 0, 0, 526, 502, 515, 522, 509, 519, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 509, 514, 531, 520, 454, 470, 529, 470, 1, 0, 0, 0, 0, 0, 0, 0, 499, 511, 519, 526, 482, 569, 491, 493, 0, 0, 0, 0, 0, 0, 0, 0, 531, 525, 523, 514, 534, 514, 537, 528, 9, 6, 0, 0, 0, 0, 0, 0, 543, 501, 512, 534, 499, 493, 478, 515, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4045, 4075, 4105, 3998, 4090, 4221, 4092] avg=4096.0000 max=4221 max_rank=6 min=3998 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 507, 517, 525, 521, 465, 523, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 509, 484, 516, 517, 534, 527, 6, 0, 0, 0, 0, 0, 0, 0, 536, 541, 520, 507, 504, 528, 514, 514, 20, 0, 0, 0, 0, 0, 0, 0, 546, 511, 489, 527, 531, 475, 510, 488, 0, 0, 0, 0, 0, 0, 0, 0, 495, 517, 512, 511, 573, 501, 544, 496, 1, 0, 0, 0, 0, 0, 0, 0, 514, 480, 510, 512, 524, 512, 503, 537, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 525, 489, 462, 508, 498, 493, 9, 6, 0, 0, 0, 0, 0, 0, 483, 542, 487, 494, 491, 512, 508, 499, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4126, 4184, 4077, 4150, 4092, 4025, 4034] avg=4096.0000 max=4184 max_rank=2 min=4025 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 502, 506, 545, 564, 532, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 505, 553, 492, 467, 498, 520, 483, 531, 6, 0, 0, 0, 0, 0, 0, 0, 524, 534, 530, 540, 521, 488, 496, 507, 20, 0, 0, 0, 0, 0, 0, 0, 505, 496, 484, 500, 493, 503, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 486, 492, 495, 506, 521, 498, 1, 0, 0, 0, 0, 0, 0, 0, 524, 507, 555, 491, 516, 516, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 481, 489, 471, 516, 502, 525, 503, 535, 9, 7, 0, 0, 0, 0, 0, 0, 504, 490, 539, 530, 498, 527, 562, 527, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4055, 4160, 3985, 4036, 4102, 4038, 4194] avg=4096.0000 max=4198 max_rank=0 min=3985 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 479, 530, 501, 494, 503, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 532, 550, 500, 501, 514, 540, 5, 0, 0, 0, 0, 0, 0, 0, 527, 493, 477, 541, 496, 501, 509, 475, 19, 0, 0, 0, 0, 0, 0, 0, 537, 486, 511, 552, 481, 493, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 530, 526, 525, 505, 538, 559, 560, 516, 1, 0, 0, 0, 0, 0, 0, 0, 553, 512, 481, 489, 512, 528, 556, 517, 0, 0, 0, 0, 0, 0, 0, 0, 464, 495, 491, 547, 471, 497, 522, 497, 8, 6, 0, 0, 0, 0, 0, 0, 522, 494, 528, 486, 484, 533, 522, 500, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4137, 4038, 4076, 4260, 4148, 3998, 4086] avg=4096.0000 max=4260 max_rank=4 min=3998 min_rank=6 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[495, 524, 499, 515, 530, 496, 509, 489, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 514, 499, 484, 534, 517, 486, 6, 0, 0, 0, 0, 0, 0, 0, 489, 514, 556, 497, 521, 513, 531, 511, 17, 0, 0, 0, 0, 0, 0, 0, 499, 533, 510, 499, 502, 521, 512, 521, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 521, 505, 521, 529, 518, 518, 1, 0, 0, 0, 0, 0, 0, 0, 520, 530, 533, 504, 461, 504, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 540, 529, 498, 542, 488, 473, 516, 513, 9, 6, 0, 0, 0, 0, 0, 0, 492, 497, 540, 522, 514, 502, 515, 501, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4074, 4149, 4097, 4107, 4070, 4114, 4100] avg=4096.0000 max=4149 max_rank=2 min=4057 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[480, 516, 482, 516, 509, 545, 513, 534, 0, 0, 0, 0, 0, 0, 0, 0, 512, 444, 542, 505, 510, 489, 519, 494, 6, 0, 0, 0, 0, 0, 0, 0, 467, 498, 534, 512, 485, 531, 484, 518, 18, 0, 0, 0, 0, 0, 0, 0, 554, 533, 482, 516, 539, 519, 518, 541, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 480, 529, 504, 509, 532, 517, 1, 0, 0, 0, 0, 0, 0, 0, 512, 554, 513, 510, 487, 516, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 528, 495, 558, 488, 508, 516, 499, 503, 9, 6, 0, 0, 0, 0, 0, 0, 526, 516, 520, 505, 504, 504, 484, 491, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4021, 4047, 4202, 4119, 4106, 4110, 4068] avg=4096.0000 max=4202 max_rank=3 min=4021 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[485, 538, 502, 489, 514, 461, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 533, 560, 504, 520, 506, 496, 506, 510, 9, 0, 0, 0, 0, 0, 0, 0, 521, 508, 446, 489, 546, 499, 501, 532, 24, 0, 0, 0, 0, 0, 0, 0, 501, 517, 495, 515, 449, 541, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 498, 486, 518, 505, 533, 474, 2, 0, 0, 0, 0, 0, 0, 0, 518, 557, 560, 515, 490, 496, 533, 517, 0, 0, 0, 0, 0, 0, 0, 0, 522, 458, 493, 513, 519, 531, 507, 534, 12, 11, 0, 0, 0, 0, 0, 0, 500, 517, 539, 499, 545, 531, 519, 486, 14, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4144, 4066, 4063, 4052, 4186, 4100, 4160] avg=4096.0000 max=4186 max_rank=5 min=3997 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[546, 534, 495, 494, 525, 540, 532, 508, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 551, 533, 489, 548, 547, 481, 5, 0, 0, 0, 0, 0, 0, 0, 518, 483, 503, 474, 510, 526, 486, 530, 19, 0, 0, 0, 0, 0, 0, 0, 519, 535, 497, 480, 538, 532, 524, 538, 0, 0, 0, 0, 0, 0, 0, 0, 499, 507, 504, 513, 508, 489, 514, 511, 1, 0, 0, 0, 0, 0, 0, 0, 505, 493, 478, 452, 508, 500, 520, 518, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 463, 549, 544, 529, 505, 511, 9, 6, 0, 0, 0, 0, 0, 0, 520, 553, 520, 487, 507, 505, 475, 448, 13, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4166, 4049, 4163, 4046, 3974, 4162, 4034] avg=4096.0000 max=4174 max_rank=0 min=3974 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 3968, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[514, 513, 497, 521, 526, 552, 523, 498, 522, 483, 505, 521, 552, 500, 522, 471, 507, 512, 486, 500, 523, 558, 518, 494, 534, 509, 481, 502, 495, 521, 499, 531, 537, 525, 511, 501, 538, 520, 527, 514, 497, 476, 473, 523, 505, 542, 518, 507, 510, 506, 515, 513, 494, 516, 508, 528, 518, 515, 503, 502, 502, 503, 513, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[508, 543, 532, 498, 555, 521, 532, 508, 503, 479, 517, 511, 490, 508, 491, 520, 541, 524, 514, 509, 509, 493, 510, 512, 510, 513, 526, 524, 507, 537, 517, 485, 526, 542, 520, 504, 522, 483, 482, 500, 497, 491, 528, 528, 454, 512, 514, 509, 521, 473, 510, 495, 517, 528, 504, 496, 501, 569, 532, 524, 507, 472, 539, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[506, 523, 533, 499, 533, 530, 475, 543, 496, 553, 515, 476, 517, 505, 568, 537, 516, 524, 510, 498, 521, 513, 510, 540, 514, 527, 525, 508, 552, 514, 481, 508, 533, 475, 508, 506, 509, 513, 506, 525, 492, 533, 521, 524, 509, 546, 532, 493, 483, 515, 502, 463, 503, 527, 496, 497, 485, 474, 489, 488, 549, 505, 500, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[507, 508, 498, 486, 499, 531, 514, 505, 500, 533, 566, 477, 489, 499, 528, 519, 546, 524, 501, 511, 535, 522, 525, 480, 522, 516, 475, 520, 513, 534, 523, 523, 465, 533, 502, 549, 547, 513, 487, 498, 496, 493, 539, 492, 552, 504, 501, 524, 488, 487, 527, 501, 527, 500, 543, 503, 543, 488, 486, 487, 520, 458, 545, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[525, 532, 509, 478, 542, 530, 515, 525, 538, 458, 520, 540, 494, 539, 498, 546, 480, 496, 491, 509, 473, 502, 502, 510, 511, 515, 499, 563, 536, 508, 520, 536, 484, 498, 477, 520, 507, 505, 498, 492, 537, 521, 522, 507, 555, 490, 502, 522, 531, 496, 491, 519, 504, 478, 478, 507, 513, 502, 516, 546, 502, 539, 552, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[523, 483, 526, 485, 487, 538, 498, 514, 517, 487, 522, 541, 544, 514, 540, 493, 533, 514, 496, 540, 494, 550, 499, 533, 500, 487, 526, 527, 525, 521, 472, 513, 497, 494, 524, 540, 495, 506, 511, 504, 489, 497, 519, 521, 519, 504, 535, 562, 499, 499, 527, 466, 534, 527, 506, 496, 523, 529, 498, 508, 527, 522, 461, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[512, 537, 548, 507, 519, 539, 494, 499, 490, 529, 512, 527, 512, 487, 492, 504, 509, 559, 486, 531, 510, 508, 509, 524, 514, 511, 531, 503, 554, 504, 489, 502, 497, 502, 547, 524, 530, 536, 511, 528, 479, 476, 522, 488, 510, 496, 548, 497, 484, 524, 528, 487, 509, 528, 500, 502, 544, 500, 506, 493, 490, 521, 483, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[484, 509, 473, 525, 507, 521, 525, 513, 502, 533, 538, 522, 522, 464, 521, 459, 498, 517, 498, 534, 514, 498, 521, 486, 527, 520, 506, 489, 517, 505, 515, 499, 510, 493, 493, 513, 554, 525, 533, 510, 535, 531, 522, 519, 522, 506, 495, 503, 504, 515, 516, 508, 519, 515, 523, 519, 521, 543, 477, 523, 480, 520, 531, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[507, 508, 498, 486, 499, 508, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 565, 477, 489, 499, 528, 519, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 501, 511, 535, 522, 525, 480, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 475, 520, 498, 534, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0, 465, 533, 502, 549, 546, 513, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 496, 493, 538, 492, 552, 504, 501, 524, 0, 0, 0, 0, 0, 0, 0, 0, 488, 487, 527, 501, 527, 500, 543, 503, 23, 15, 3, 0, 0, 0, 0, 0, 543, 488, 486, 487, 520, 458, 545, 541, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4110, 4139, 4111, 4093, 4100, 4117, 4073] avg=4096.0000 max=4139 max_rank=2 min=4025 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[525, 532, 509, 478, 542, 507, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 538, 458, 519, 540, 494, 539, 498, 546, 0, 0, 0, 0, 0, 0, 0, 0, 480, 492, 491, 509, 473, 502, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 499, 563, 521, 508, 520, 536, 0, 0, 0, 0, 0, 0, 0, 0, 484, 498, 477, 520, 506, 505, 498, 492, 0, 0, 0, 0, 0, 0, 0, 0, 537, 521, 521, 507, 555, 490, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 491, 519, 504, 478, 478, 507, 23, 15, 2, 0, 0, 0, 0, 0, 513, 502, 516, 546, 502, 539, 552, 517, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4132, 3959, 4173, 3980, 4155, 4044, 4192] avg=4096.0000 max=4192 max_rank=7 min=3959 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 483, 526, 485, 487, 512, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 517, 487, 519, 541, 544, 514, 540, 493, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 496, 540, 494, 550, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 500, 487, 526, 527, 507, 521, 472, 513, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 524, 540, 491, 506, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 497, 517, 521, 519, 504, 535, 562, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 527, 466, 534, 527, 506, 496, 26, 18, 5, 0, 0, 0, 0, 0, 523, 529, 498, 508, 527, 522, 461, 487, 3, 4, 3, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4155, 4151, 4053, 4067, 4144, 4103, 4067] avg=4096.0000 max=4155 max_rank=1 min=4028 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[484, 509, 473, 525, 507, 499, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 537, 522, 522, 464, 521, 459, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 498, 534, 514, 498, 521, 486, 0, 0, 0, 0, 0, 0, 0, 0, 527, 520, 506, 489, 502, 505, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0, 510, 493, 493, 513, 553, 525, 533, 510, 0, 0, 0, 0, 0, 0, 0, 0, 535, 531, 521, 519, 522, 506, 495, 503, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 516, 508, 519, 515, 523, 519, 22, 15, 3, 0, 0, 0, 0, 0, 521, 543, 477, 523, 480, 520, 531, 528, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4060, 4061, 4063, 4130, 4132, 4159, 4128] avg=4096.0000 max=4159 max_rank=6 min=4035 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 533, 499, 533, 507, 475, 543, 0, 0, 0, 0, 0, 0, 0, 0, 496, 553, 514, 476, 517, 505, 568, 537, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 510, 498, 521, 513, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 514, 527, 525, 508, 536, 514, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 533, 475, 508, 506, 508, 513, 506, 525, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 520, 524, 509, 546, 532, 493, 0, 0, 0, 0, 0, 0, 0, 0, 483, 515, 502, 463, 503, 527, 496, 497, 23, 16, 3, 0, 0, 0, 0, 0, 485, 474, 489, 488, 549, 505, 500, 497, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4166, 4127, 4113, 4074, 4149, 4028, 3992] avg=4096.0000 max=4166 max_rank=1 min=3992 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 513, 497, 521, 526, 529, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 504, 521, 552, 500, 522, 471, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 486, 500, 523, 558, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 534, 509, 481, 502, 481, 521, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 511, 501, 537, 520, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 497, 476, 473, 523, 505, 542, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 515, 513, 494, 516, 508, 528, 23, 14, 3, 0, 0, 0, 0, 0, 518, 515, 503, 502, 502, 503, 513, 518, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4075, 4093, 4058, 4172, 4041, 4130, 4078] avg=4096.0000 max=4172 max_rank=4 min=4041 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[512, 537, 548, 507, 519, 516, 494, 499, 0, 0, 0, 0, 0, 0, 0, 0, 490, 529, 511, 527, 512, 487, 492, 504, 0, 0, 0, 0, 0, 0, 0, 0, 509, 554, 486, 531, 510, 508, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 514, 511, 531, 503, 538, 504, 489, 502, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 547, 524, 529, 536, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 479, 476, 521, 488, 510, 496, 548, 497, 0, 0, 0, 0, 0, 0, 0, 0, 484, 524, 528, 487, 509, 528, 500, 502, 23, 16, 3, 0, 0, 0, 0, 0, 544, 500, 506, 493, 490, 521, 483, 526, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4052, 4131, 4092, 4174, 4015, 4104, 4068] avg=4096.0000 max=4174 max_rank=4 min=4015 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 543, 532, 498, 555, 499, 532, 508, 0, 0, 0, 0, 0, 0, 0, 0, 503, 479, 516, 511, 490, 508, 491, 520, 0, 0, 0, 0, 0, 0, 0, 0, 541, 519, 514, 509, 509, 493, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 526, 524, 493, 537, 517, 485, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 520, 504, 521, 483, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0, 497, 491, 527, 528, 454, 512, 514, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 473, 510, 495, 517, 528, 504, 496, 22, 14, 3, 0, 0, 0, 0, 0, 501, 569, 532, 524, 507, 472, 539, 521, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4018, 4107, 4105, 4078, 4032, 4083, 4170] avg=4096.0000 max=4175 max_rank=0 min=4018 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[523, 535, 511, 527, 535, 521, 540, 513, 526, 465, 512, 515, 532, 478, 515, 535, 507, 486, 517, 510, 497, 519, 573, 519, 509, 477, 555, 486, 535, 552, 521, 504, 496, 480, 485, 481, 493, 488, 497, 482, 490, 510, 497, 502, 515, 571, 482, 505, 523, 479, 558, 536, 537, 536, 491, 506, 509, 512, 502, 532, 495, 503, 516, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[506, 503, 518, 545, 507, 504, 514, 495, 526, 514, 501, 492, 526, 504, 467, 495, 537, 510, 518, 491, 493, 510, 482, 509, 517, 509, 522, 513, 529, 521, 514, 536, 529, 488, 526, 517, 505, 542, 536, 516, 566, 514, 523, 457, 514, 495, 501, 524, 506, 515, 526, 520, 512, 510, 485, 514, 503, 519, 578, 477, 488, 524, 492, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[493, 468, 513, 540, 527, 523, 478, 537, 503, 551, 556, 509, 490, 580, 472, 538, 490, 530, 527, 518, 505, 523, 503, 526, 492, 499, 529, 533, 500, 537, 483, 508, 494, 457, 526, 461, 499, 535, 504, 506, 524, 489, 502, 530, 499, 494, 507, 474, 520, 525, 503, 526, 491, 519, 511, 515, 505, 535, 531, 534, 526, 505, 525, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[513, 493, 516, 523, 543, 501, 515, 536, 493, 520, 544, 527, 498, 489, 533, 523, 541, 532, 546, 498, 534, 508, 513, 475, 515, 482, 511, 500, 573, 510, 500, 512, 539, 501, 530, 519, 495, 554, 523, 500, 488, 545, 493, 532, 505, 480, 517, 473, 500, 518, 515, 515, 522, 502, 466, 498, 468, 533, 547, 486, 472, 467, 512, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[562, 478, 505, 512, 470, 497, 501, 479, 475, 546, 492, 533, 503, 509, 486, 483, 537, 517, 546, 541, 514, 521, 477, 510, 549, 531, 524, 519, 521, 514, 509, 537, 489, 554, 539, 534, 490, 508, 520, 523, 519, 530, 522, 493, 516, 553, 519, 532, 489, 504, 515, 459, 531, 470, 508, 513, 469, 505, 534, 489, 505, 502, 512, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[480, 518, 551, 500, 518, 520, 538, 496, 518, 499, 492, 506, 484, 505, 500, 521, 499, 507, 513, 544, 528, 469, 491, 453, 524, 505, 487, 565, 563, 529, 495, 535, 492, 487, 573, 497, 506, 480, 517, 504, 498, 502, 562, 538, 488, 515, 511, 488, 521, 532, 509, 516, 512, 505, 520, 536, 528, 502, 497, 531, 544, 523, 475, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[495, 486, 517, 484, 490, 523, 570, 522, 545, 544, 497, 519, 533, 487, 514, 493, 491, 542, 505, 508, 504, 486, 526, 507, 514, 501, 563, 488, 506, 516, 502, 474, 522, 542, 541, 544, 516, 505, 488, 495, 521, 520, 494, 499, 501, 487, 504, 526, 508, 537, 489, 497, 500, 570, 500, 505, 502, 485, 530, 492, 530, 541, 520, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[506, 519, 493, 505, 504, 543, 513, 484, 516, 494, 504, 509, 499, 553, 527, 524, 510, 496, 510, 507, 511, 475, 520, 529, 500, 532, 508, 509, 537, 534, 469, 512, 537, 487, 507, 506, 539, 536, 535, 477, 538, 522, 504, 478, 498, 541, 481, 524, 523, 507, 462, 535, 483, 527, 503, 500, 518, 504, 521, 532, 520, 527, 519, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 535, 511, 527, 535, 521, 538, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 465, 512, 515, 532, 478, 515, 535, 2, 0, 0, 0, 0, 0, 0, 0, 507, 486, 517, 510, 497, 519, 573, 519, 2, 0, 0, 0, 0, 0, 0, 0, 509, 477, 555, 486, 501, 552, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 485, 481, 493, 488, 497, 482, 9, 0, 0, 0, 0, 0, 0, 0, 490, 510, 497, 502, 515, 571, 482, 505, 15, 0, 0, 0, 0, 0, 0, 0, 523, 479, 558, 536, 537, 536, 491, 506, 10, 0, 0, 0, 0, 0, 0, 0, 509, 512, 500, 532, 495, 503, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 4080, 4130, 4105, 3911, 4087, 4176, 4076] avg=4096.0000 max=4203 max_rank=0 min=3911 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[480, 518, 551, 500, 518, 520, 536, 496, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 492, 506, 484, 505, 500, 521, 2, 0, 0, 0, 0, 0, 0, 0, 499, 507, 513, 544, 528, 469, 491, 453, 2, 0, 0, 0, 0, 0, 0, 0, 524, 505, 487, 565, 528, 529, 495, 535, 0, 0, 0, 0, 0, 0, 0, 0, 492, 487, 573, 497, 506, 480, 517, 504, 9, 0, 0, 0, 0, 0, 0, 0, 498, 502, 562, 538, 488, 515, 511, 488, 16, 0, 0, 0, 0, 0, 0, 0, 521, 532, 509, 516, 512, 505, 520, 536, 10, 0, 0, 0, 0, 0, 0, 0, 528, 502, 495, 531, 544, 523, 475, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4027, 4006, 4168, 4065, 4118, 4161, 4104] avg=4096.0000 max=4168 max_rank=3 min=4006 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 503, 518, 545, 507, 504, 512, 495, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 501, 492, 526, 504, 467, 495, 2, 0, 0, 0, 0, 0, 0, 0, 537, 510, 518, 491, 493, 510, 482, 509, 3, 0, 0, 0, 0, 0, 0, 0, 517, 509, 522, 513, 495, 521, 514, 536, 0, 0, 0, 0, 0, 0, 0, 0, 529, 488, 526, 517, 505, 542, 536, 516, 9, 0, 0, 0, 0, 0, 0, 0, 566, 514, 523, 457, 514, 495, 501, 524, 15, 0, 0, 0, 0, 0, 0, 0, 506, 515, 526, 520, 512, 510, 485, 514, 10, 0, 0, 0, 0, 0, 0, 0, 503, 519, 575, 477, 488, 524, 492, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4027, 4053, 4127, 4168, 4109, 4098, 4096] avg=4096.0000 max=4168 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 468, 513, 540, 527, 523, 476, 537, 0, 0, 0, 0, 0, 0, 0, 0, 503, 551, 556, 509, 490, 580, 472, 538, 2, 0, 0, 0, 0, 0, 0, 0, 490, 530, 527, 518, 505, 523, 503, 526, 2, 0, 0, 0, 0, 0, 0, 0, 492, 499, 529, 533, 469, 537, 483, 508, 0, 0, 0, 0, 0, 0, 0, 0, 494, 457, 526, 461, 499, 535, 504, 506, 8, 0, 0, 0, 0, 0, 0, 0, 524, 489, 502, 530, 499, 494, 507, 474, 14, 0, 0, 0, 0, 0, 0, 0, 520, 525, 503, 526, 491, 519, 511, 515, 9, 0, 0, 0, 0, 0, 0, 0, 505, 535, 529, 534, 526, 505, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4201, 4124, 4050, 3990, 4033, 4119, 4174] avg=4096.0000 max=4201 max_rank=1 min=3990 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 493, 516, 523, 543, 501, 513, 536, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 544, 527, 498, 489, 533, 523, 2, 0, 0, 0, 0, 0, 0, 0, 541, 532, 546, 498, 534, 508, 513, 475, 2, 0, 0, 0, 0, 0, 0, 0, 515, 482, 511, 500, 537, 510, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 539, 501, 530, 519, 495, 554, 523, 500, 9, 0, 0, 0, 0, 0, 0, 0, 488, 545, 493, 532, 505, 480, 517, 473, 16, 0, 0, 0, 0, 0, 0, 0, 500, 518, 515, 515, 522, 502, 466, 498, 11, 0, 0, 0, 0, 0, 0, 0, 468, 533, 545, 486, 472, 467, 512, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4129, 4149, 4067, 4170, 4049, 4047, 4019] avg=4096.0000 max=4170 max_rank=4 min=4019 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[562, 478, 505, 512, 470, 497, 499, 479, 0, 0, 0, 0, 0, 0, 0, 0, 475, 546, 492, 533, 503, 509, 486, 483, 2, 0, 0, 0, 0, 0, 0, 0, 537, 517, 546, 541, 514, 521, 477, 510, 2, 0, 0, 0, 0, 0, 0, 0, 549, 531, 524, 519, 487, 514, 509, 537, 0, 0, 0, 0, 0, 0, 0, 0, 489, 554, 539, 534, 490, 508, 520, 523, 9, 0, 0, 0, 0, 0, 0, 0, 519, 530, 522, 493, 516, 553, 519, 532, 15, 0, 0, 0, 0, 0, 0, 0, 489, 504, 515, 459, 531, 470, 508, 513, 10, 0, 0, 0, 0, 0, 0, 0, 469, 505, 532, 489, 505, 502, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4029, 4165, 4170, 4166, 4199, 3999, 4038] avg=4096.0000 max=4199 max_rank=5 min=3999 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 486, 517, 484, 490, 523, 557, 522, 0, 0, 0, 0, 0, 0, 0, 0, 545, 544, 497, 519, 533, 487, 514, 493, 9, 6, 0, 0, 0, 0, 0, 0, 491, 542, 505, 508, 504, 486, 526, 507, 8, 4, 0, 0, 0, 0, 0, 0, 514, 501, 563, 488, 456, 516, 502, 474, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 541, 544, 516, 505, 488, 495, 12, 0, 0, 0, 0, 0, 0, 0, 521, 520, 494, 499, 501, 487, 504, 526, 20, 0, 0, 0, 0, 0, 0, 0, 508, 537, 489, 497, 500, 570, 500, 505, 12, 0, 0, 0, 0, 0, 0, 0, 502, 485, 522, 492, 530, 541, 520, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4147, 4081, 4014, 4165, 4072, 4118, 4097] avg=4096.0000 max=4165 max_rank=4 min=4014 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 519, 493, 505, 504, 543, 511, 484, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 504, 509, 499, 553, 527, 524, 2, 0, 0, 0, 0, 0, 0, 0, 510, 496, 510, 507, 511, 475, 520, 529, 2, 0, 0, 0, 0, 0, 0, 0, 500, 532, 508, 509, 503, 534, 469, 512, 0, 0, 0, 0, 0, 0, 0, 0, 537, 487, 507, 506, 539, 536, 535, 477, 9, 0, 0, 0, 0, 0, 0, 0, 538, 522, 504, 478, 498, 541, 481, 524, 15, 0, 0, 0, 0, 0, 0, 0, 523, 507, 462, 535, 483, 527, 503, 500, 10, 0, 0, 0, 0, 0, 0, 0, 518, 504, 519, 532, 520, 527, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4128, 4060, 4067, 4133, 4101, 4050, 4164] avg=4096.0000 max=4164 max_rank=7 min=4050 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4352, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[520, 500, 518, 488, 508, 537, 527, 483, 508, 486, 522, 502, 508, 522, 494, 524, 512, 502, 499, 494, 477, 481, 546, 513, 546, 509, 493, 490, 498, 536, 520, 499, 502, 517, 496, 506, 512, 514, 511, 517, 477, 544, 486, 495, 466, 512, 519, 508, 535, 536, 500, 533, 510, 518, 539, 520, 506, 551, 517, 535, 529, 543, 514, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[528, 518, 511, 513, 527, 519, 494, 504, 494, 485, 514, 500, 515, 521, 525, 506, 502, 506, 460, 538, 530, 532, 478, 528, 530, 517, 486, 525, 499, 503, 503, 506, 500, 517, 485, 525, 529, 493, 497, 476, 523, 503, 516, 498, 509, 510, 512, 544, 522, 514, 531, 528, 521, 528, 542, 532, 498, 523, 505, 518, 468, 528, 509, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[541, 535, 517, 519, 472, 503, 506, 486, 534, 498, 496, 528, 520, 482, 474, 527, 493, 493, 498, 509, 535, 531, 514, 535, 488, 546, 504, 496, 539, 499, 513, 524, 520, 534, 533, 523, 536, 553, 496, 506, 517, 503, 521, 524, 479, 501, 511, 532, 556, 525, 504, 503, 482, 527, 508, 508, 479, 487, 513, 541, 513, 515, 488, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[495, 502, 484, 518, 508, 521, 546, 528, 524, 499, 498, 545, 478, 542, 515, 495, 508, 528, 492, 512, 502, 486, 519, 516, 479, 513, 524, 534, 502, 540, 482, 523, 504, 523, 529, 517, 517, 536, 505, 522, 497, 485, 515, 519, 509, 506, 505, 520, 513, 510, 506, 520, 523, 535, 527, 507, 520, 530, 512, 484, 481, 530, 515, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[518, 530, 459, 503, 514, 523, 506, 522, 523, 516, 519, 552, 506, 503, 503, 517, 513, 508, 486, 507, 513, 538, 499, 509, 483, 508, 510, 508, 513, 499, 520, 483, 518, 514, 512, 539, 520, 516, 495, 544, 489, 459, 520, 535, 510, 462, 525, 547, 504, 483, 539, 516, 522, 529, 512, 498, 541, 492, 536, 535, 520, 459, 503, 563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[546, 510, 510, 493, 514, 519, 593, 510, 480, 519, 487, 531, 514, 496, 497, 552, 523, 529, 525, 507, 506, 490, 537, 496, 543, 465, 539, 522, 477, 541, 477, 507, 484, 550, 501, 564, 506, 516, 479, 476, 515, 497, 494, 509, 504, 493, 515, 511, 535, 528, 489, 512, 490, 518, 502, 510, 498, 534, 524, 523, 507, 503, 527, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[502, 515, 509, 527, 486, 557, 538, 507, 515, 486, 520, 494, 562, 534, 481, 496, 498, 519, 500, 514, 527, 545, 484, 531, 495, 477, 496, 525, 479, 520, 535, 488, 483, 524, 477, 512, 513, 514, 522, 544, 514, 499, 521, 491, 519, 496, 510, 527, 514, 496, 513, 578, 499, 493, 518, 504, 536, 514, 536, 488, 542, 501, 493, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[535, 528, 525, 483, 529, 521, 509, 517, 519, 484, 490, 492, 511, 475, 516, 489, 517, 448, 515, 523, 503, 531, 538, 497, 499, 499, 487, 530, 536, 507, 525, 506, 546, 508, 527, 509, 508, 512, 507, 513, 484, 513, 518, 508, 488, 499, 510, 515, 522, 471, 507, 522, 506, 544, 511, 510, 512, 532, 543, 542, 517, 527, 521, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 518, 511, 513, 527, 519, 475, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 485, 514, 500, 515, 521, 525, 506, 25, 0, 0, 0, 0, 0, 0, 0, 502, 506, 460, 538, 530, 532, 478, 528, 15, 0, 0, 0, 0, 0, 0, 0, 530, 517, 486, 525, 499, 503, 503, 506, 19, 2, 1, 0, 0, 0, 0, 0, 500, 517, 485, 509, 529, 493, 497, 476, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 516, 498, 509, 510, 512, 544, 36, 1, 0, 0, 0, 0, 0, 0, 522, 514, 531, 492, 521, 528, 542, 532, 0, 0, 0, 0, 0, 0, 0, 0, 498, 523, 477, 518, 468, 528, 509, 547, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 4089, 4091, 4006, 4152, 4182, 4068] avg=4096.0000 max=4182 max_rank=6 min=4006 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 500, 518, 488, 508, 537, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 522, 502, 508, 522, 494, 524, 25, 0, 0, 0, 0, 0, 0, 0, 512, 502, 499, 494, 477, 481, 546, 513, 14, 0, 0, 0, 0, 0, 0, 0, 546, 509, 493, 490, 498, 536, 520, 499, 20, 2, 1, 0, 0, 0, 0, 0, 502, 517, 496, 491, 512, 514, 511, 517, 0, 0, 0, 0, 0, 0, 0, 0, 477, 544, 486, 495, 466, 512, 519, 508, 36, 1, 0, 0, 0, 0, 0, 0, 535, 536, 500, 497, 510, 518, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 506, 551, 489, 535, 529, 543, 514, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4091, 4038, 4114, 4060, 4044, 4155, 4205] avg=4096.0000 max=4205 max_rank=7 min=4038 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[546, 510, 510, 493, 514, 519, 570, 510, 0, 0, 0, 0, 0, 0, 0, 0, 480, 519, 487, 531, 514, 496, 497, 552, 26, 0, 0, 0, 0, 0, 0, 0, 523, 529, 525, 507, 506, 490, 537, 496, 16, 0, 0, 0, 0, 0, 0, 0, 543, 465, 539, 522, 477, 541, 477, 507, 23, 3, 1, 0, 0, 0, 0, 0, 484, 550, 501, 547, 506, 516, 479, 476, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 494, 509, 504, 493, 515, 511, 35, 1, 0, 0, 0, 0, 0, 0, 535, 528, 489, 477, 490, 518, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 498, 534, 494, 523, 507, 503, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4102, 4129, 4098, 4059, 4074, 4049, 4085] avg=4096.0000 max=4172 max_rank=0 min=4049 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[541, 535, 517, 519, 472, 503, 487, 486, 0, 0, 0, 0, 0, 0, 0, 0, 534, 498, 496, 528, 520, 482, 474, 527, 25, 0, 0, 0, 0, 0, 0, 0, 493, 493, 498, 509, 535, 531, 514, 535, 15, 0, 0, 0, 0, 0, 0, 0, 488, 546, 504, 496, 539, 499, 513, 524, 19, 2, 1, 0, 0, 0, 0, 0, 520, 534, 533, 507, 536, 553, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 521, 524, 479, 501, 511, 532, 34, 1, 0, 0, 0, 0, 0, 0, 556, 525, 504, 469, 482, 527, 508, 508, 0, 0, 0, 0, 0, 0, 0, 0, 479, 487, 485, 541, 513, 515, 488, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4084, 4123, 4131, 4185, 4123, 4079, 3983] avg=4096.0000 max=4185 max_rank=4 min=3983 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[495, 502, 484, 518, 508, 521, 525, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 498, 545, 478, 542, 515, 495, 25, 0, 0, 0, 0, 0, 0, 0, 508, 528, 492, 512, 502, 486, 519, 516, 15, 0, 0, 0, 0, 0, 0, 0, 479, 513, 524, 534, 502, 540, 482, 523, 21, 2, 1, 0, 0, 0, 0, 0, 504, 523, 529, 501, 517, 536, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 515, 519, 509, 506, 505, 520, 35, 1, 0, 0, 0, 0, 0, 0, 513, 510, 506, 485, 523, 535, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 520, 530, 484, 484, 481, 530, 515, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4121, 4078, 4121, 4137, 4092, 4106, 4032] avg=4096.0000 max=4137 max_rank=4 min=4032 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 530, 459, 503, 514, 523, 487, 522, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 519, 552, 506, 503, 503, 517, 26, 0, 0, 0, 0, 0, 0, 0, 513, 508, 486, 507, 513, 538, 499, 509, 15, 0, 0, 0, 0, 0, 0, 0, 483, 508, 510, 508, 513, 499, 520, 483, 19, 3, 1, 0, 0, 0, 0, 0, 518, 514, 512, 523, 520, 516, 495, 544, 0, 0, 0, 0, 0, 0, 0, 0, 489, 459, 520, 535, 510, 462, 525, 547, 35, 1, 0, 0, 0, 0, 0, 0, 504, 483, 539, 481, 522, 529, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 541, 492, 506, 535, 520, 459, 503, 563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4165, 4088, 4047, 4142, 4083, 4068, 4119] avg=4096.0000 max=4165 max_rank=1 min=4047 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 515, 509, 527, 486, 557, 513, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 520, 494, 562, 534, 481, 496, 30, 0, 0, 0, 0, 0, 0, 0, 498, 519, 500, 514, 527, 545, 484, 531, 19, 0, 0, 0, 0, 0, 0, 0, 495, 477, 496, 525, 479, 520, 535, 488, 25, 7, 6, 0, 0, 0, 0, 0, 483, 524, 477, 487, 513, 514, 522, 544, 0, 0, 0, 0, 0, 0, 0, 0, 514, 499, 521, 491, 519, 496, 510, 527, 44, 8, 0, 0, 0, 0, 0, 0, 514, 496, 513, 534, 499, 493, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 491, 488, 542, 501, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4118, 4137, 4053, 4064, 4129, 4071, 4080] avg=4096.0000 max=4137 max_rank=2 min=4053 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 528, 525, 483, 529, 521, 489, 517, 0, 0, 0, 0, 0, 0, 0, 0, 519, 484, 490, 492, 511, 475, 516, 489, 26, 0, 0, 0, 0, 0, 0, 0, 517, 448, 515, 523, 503, 531, 538, 497, 14, 0, 0, 0, 0, 0, 0, 0, 499, 499, 487, 530, 536, 507, 525, 506, 20, 3, 1, 0, 0, 0, 0, 0, 546, 508, 527, 494, 508, 512, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 484, 513, 518, 508, 488, 499, 510, 515, 35, 1, 0, 0, 0, 0, 0, 0, 522, 471, 507, 487, 506, 544, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 513, 542, 517, 527, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4002, 4086, 4113, 4115, 4071, 4058, 4196] avg=4096.0000 max=4196 max_rank=7 min=4002 min_rank=1 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[499, 493, 539, 523, 463, 506, 537, 514, 507, 499, 495, 498, 509, 506, 531, 514, 541, 529, 498, 518, 478, 470, 523, 490, 518, 524, 501, 502, 478, 535, 499, 492, 514, 568, 504, 528, 537, 554, 475, 541, 499, 525, 476, 551, 535, 542, 568, 527, 518, 509, 535, 450, 475, 530, 507, 534, 461, 520, 506, 505, 469, 538, 532, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[503, 481, 503, 495, 522, 493, 509, 513, 527, 496, 551, 547, 503, 483, 547, 508, 533, 507, 508, 517, 479, 498, 546, 542, 560, 494, 481, 528, 542, 499, 491, 506, 561, 507, 508, 495, 511, 483, 507, 512, 517, 522, 468, 529, 499, 468, 520, 504, 518, 535, 556, 536, 518, 501, 502, 491, 505, 501, 533, 496, 515, 513, 534, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[481, 517, 504, 488, 534, 543, 490, 523, 531, 505, 511, 533, 503, 496, 498, 552, 529, 519, 498, 534, 521, 514, 512, 494, 486, 568, 525, 520, 501, 537, 522, 482, 535, 516, 520, 502, 474, 495, 489, 515, 487, 543, 507, 486, 515, 527, 501, 488, 511, 517, 496, 508, 523, 479, 491, 546, 498, 514, 508, 514, 534, 517, 527, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[469, 504, 518, 537, 495, 517, 542, 545, 519, 487, 518, 504, 553, 517, 519, 522, 527, 512, 488, 501, 528, 522, 526, 546, 500, 534, 476, 512, 523, 512, 509, 508, 496, 528, 488, 489, 531, 506, 538, 497, 491, 495, 538, 494, 533, 489, 543, 520, 476, 524, 486, 487, 463, 530, 533, 499, 469, 523, 522, 512, 521, 502, 556, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[555, 545, 472, 510, 565, 521, 495, 502, 529, 507, 509, 479, 508, 545, 494, 519, 484, 531, 527, 479, 517, 537, 501, 487, 553, 503, 526, 514, 524, 497, 507, 484, 510, 549, 532, 539, 514, 484, 524, 511, 518, 512, 570, 514, 494, 472, 487, 522, 495, 554, 484, 499, 519, 459, 483, 525, 503, 528, 469, 530, 491, 500, 550, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[544, 533, 512, 490, 545, 486, 523, 549, 516, 497, 532, 489, 545, 507, 504, 499, 476, 497, 487, 499, 502, 509, 465, 502, 497, 501, 530, 559, 498, 496, 523, 511, 516, 538, 480, 522, 538, 545, 505, 495, 522, 504, 485, 522, 495, 484, 548, 517, 506, 537, 483, 507, 495, 505, 558, 501, 542, 497, 517, 493, 522, 501, 549, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[495, 565, 476, 534, 524, 478, 549, 507, 467, 497, 521, 511, 536, 515, 478, 523, 508, 521, 509, 529, 513, 506, 536, 515, 513, 507, 515, 473, 498, 499, 545, 521, 523, 491, 531, 490, 481, 516, 500, 542, 503, 532, 493, 510, 496, 516, 516, 495, 538, 533, 496, 518, 507, 504, 510, 503, 516, 485, 547, 531, 524, 549, 486, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[494, 486, 502, 526, 526, 516, 514, 505, 522, 506, 503, 525, 464, 526, 529, 489, 498, 517, 520, 514, 531, 544, 489, 511, 514, 491, 515, 501, 536, 535, 542, 487, 499, 482, 519, 512, 495, 514, 522, 522, 495, 507, 514, 512, 531, 464, 577, 503, 518, 470, 476, 538, 504, 515, 526, 522, 535, 499, 531, 570, 483, 502, 511, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 493, 539, 523, 456, 506, 537, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 499, 494, 498, 509, 506, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 541, 529, 498, 518, 478, 470, 523, 490, 6, 0, 0, 0, 0, 0, 0, 0, 517, 524, 501, 502, 478, 535, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 514, 555, 504, 528, 537, 554, 475, 541, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 476, 551, 535, 542, 568, 527, 1, 1, 0, 0, 0, 0, 0, 0, 518, 509, 535, 450, 475, 530, 507, 534, 13, 11, 1, 0, 0, 0, 0, 0, 461, 520, 506, 505, 469, 538, 521, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4058, 4053, 4048, 4208, 4225, 4083, 4026] avg=4096.0000 max=4225 max_rank=5 min=4026 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[469, 504, 518, 537, 488, 517, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 519, 487, 517, 504, 553, 517, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 527, 512, 488, 501, 528, 522, 526, 546, 6, 0, 0, 0, 0, 0, 0, 0, 499, 534, 476, 512, 523, 512, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 488, 489, 531, 506, 538, 497, 0, 0, 0, 0, 0, 0, 0, 0, 491, 495, 538, 494, 533, 489, 543, 520, 1, 1, 0, 0, 0, 0, 0, 0, 476, 524, 486, 487, 463, 530, 533, 499, 12, 11, 1, 0, 0, 0, 0, 0, 469, 523, 522, 512, 521, 502, 545, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4138, 4156, 4073, 4061, 4105, 4022, 4093] avg=4096.0000 max=4156 max_rank=2 min=4022 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 481, 503, 495, 514, 493, 509, 513, 0, 0, 0, 0, 0, 0, 0, 0, 527, 496, 550, 547, 503, 483, 547, 508, 0, 0, 0, 0, 0, 0, 0, 0, 533, 507, 508, 517, 479, 498, 546, 542, 7, 0, 0, 0, 0, 0, 0, 0, 559, 494, 481, 528, 542, 499, 491, 506, 0, 0, 0, 0, 0, 0, 0, 0, 561, 496, 508, 495, 511, 483, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 468, 529, 499, 468, 520, 504, 1, 1, 0, 0, 0, 0, 0, 0, 518, 535, 556, 536, 518, 501, 502, 491, 11, 11, 1, 0, 0, 0, 0, 0, 505, 501, 533, 496, 515, 513, 523, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4161, 4137, 4100, 4073, 4029, 4180, 4077] avg=4096.0000 max=4180 max_rank=6 min=4011 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 517, 504, 488, 526, 543, 490, 523, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 510, 533, 503, 496, 498, 552, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 498, 534, 521, 514, 512, 494, 7, 0, 0, 0, 0, 0, 0, 0, 485, 568, 525, 520, 501, 537, 522, 482, 0, 0, 0, 0, 0, 0, 0, 0, 535, 505, 520, 502, 474, 495, 489, 515, 0, 0, 0, 0, 0, 0, 0, 0, 487, 543, 507, 486, 515, 527, 501, 488, 1, 1, 0, 0, 0, 0, 0, 0, 511, 517, 496, 508, 523, 479, 491, 546, 11, 11, 1, 0, 0, 0, 0, 0, 498, 514, 508, 514, 534, 517, 516, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4128, 4128, 4140, 4035, 4056, 4094, 4115] avg=4096.0000 max=4140 max_rank=3 min=4035 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[555, 545, 472, 510, 557, 521, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 508, 479, 508, 545, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 484, 531, 527, 479, 517, 537, 501, 487, 7, 0, 0, 0, 0, 0, 0, 0, 552, 503, 526, 514, 524, 497, 507, 484, 0, 0, 0, 0, 0, 0, 0, 0, 510, 537, 532, 539, 514, 484, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 570, 514, 494, 472, 487, 522, 1, 1, 0, 0, 0, 0, 0, 0, 495, 554, 484, 499, 519, 459, 483, 525, 12, 11, 1, 0, 0, 0, 0, 0, 503, 528, 469, 530, 491, 500, 539, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4089, 4070, 4107, 4151, 4091, 4042, 4061] avg=4096.0000 max=4157 max_rank=0 min=4042 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[544, 533, 512, 490, 537, 486, 523, 549, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 531, 489, 545, 507, 504, 499, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 487, 499, 502, 509, 465, 502, 7, 0, 0, 0, 0, 0, 0, 0, 496, 501, 530, 559, 498, 496, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 480, 522, 538, 545, 505, 495, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 485, 522, 495, 484, 548, 517, 1, 1, 0, 0, 0, 0, 0, 0, 506, 537, 483, 507, 495, 505, 558, 501, 12, 11, 1, 0, 0, 0, 0, 0, 542, 497, 517, 493, 522, 501, 538, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4088, 3944, 4114, 4127, 4079, 4116, 4126] avg=4096.0000 max=4174 max_rank=0 min=3944 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[494, 486, 502, 526, 518, 516, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 502, 525, 464, 526, 529, 489, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 520, 514, 531, 544, 489, 511, 7, 0, 0, 0, 0, 0, 0, 0, 513, 491, 515, 501, 536, 535, 542, 487, 0, 0, 0, 0, 0, 0, 0, 0, 499, 471, 519, 512, 495, 514, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 514, 512, 531, 464, 577, 503, 1, 1, 0, 0, 0, 0, 0, 0, 518, 470, 476, 538, 504, 515, 526, 522, 11, 10, 1, 0, 0, 0, 0, 0, 535, 499, 531, 570, 483, 502, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4063, 4131, 4120, 4054, 4105, 4091, 4143] avg=4096.0000 max=4143 max_rank=7 min=4054 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[495, 565, 476, 534, 502, 478, 549, 507, 0, 0, 0, 0, 0, 0, 0, 0, 467, 497, 516, 511, 536, 515, 478, 523, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 509, 529, 513, 506, 536, 515, 12, 0, 0, 0, 0, 0, 0, 0, 508, 507, 515, 473, 498, 499, 545, 521, 0, 0, 0, 0, 0, 0, 0, 0, 523, 476, 531, 490, 481, 516, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 493, 510, 496, 516, 516, 495, 5, 5, 7, 0, 0, 0, 0, 0, 538, 533, 496, 518, 507, 504, 510, 503, 15, 13, 3, 0, 0, 0, 0, 0, 516, 485, 547, 531, 524, 549, 473, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4043, 4149, 4066, 4059, 4078, 4140, 4127] avg=4096.0000 max=4149 max_rank=2 min=4043 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4352, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[551, 477, 551, 523, 490, 503, 503, 475, 523, 489, 514, 478, 522, 503, 506, 525, 512, 500, 506, 527, 523, 511, 514, 545, 498, 519, 508, 503, 517, 520, 527, 481, 517, 517, 498, 483, 539, 548, 517, 478, 528, 512, 488, 489, 508, 543, 518, 515, 563, 555, 551, 533, 525, 497, 510, 491, 506, 486, 509, 511, 460, 526, 507, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[507, 523, 489, 487, 552, 515, 526, 483, 496, 525, 506, 507, 513, 503, 526, 529, 510, 550, 499, 592, 525, 525, 496, 529, 531, 480, 496, 492, 533, 461, 507, 526, 555, 487, 525, 474, 510, 518, 487, 505, 493, 496, 494, 477, 525, 542, 494, 535, 518, 538, 507, 479, 546, 513, 487, 508, 515, 510, 538, 513, 513, 521, 514, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[523, 483, 482, 523, 552, 451, 527, 505, 531, 525, 509, 503, 497, 490, 512, 494, 523, 497, 501, 541, 536, 568, 508, 504, 521, 534, 501, 530, 468, 500, 535, 516, 498, 511, 504, 490, 519, 530, 533, 536, 520, 520, 514, 497, 489, 501, 525, 513, 475, 518, 540, 541, 539, 508, 555, 486, 503, 512, 499, 538, 470, 502, 490, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[524, 482, 527, 520, 489, 493, 508, 500, 500, 497, 524, 561, 526, 508, 546, 549, 489, 523, 528, 539, 505, 477, 481, 493, 483, 465, 542, 553, 522, 474, 517, 508, 531, 550, 513, 517, 548, 534, 463, 544, 505, 508, 524, 496, 490, 478, 472, 481, 534, 535, 518, 517, 542, 523, 516, 527, 516, 514, 513, 479, 509, 534, 465, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[515, 510, 472, 491, 492, 498, 492, 491, 519, 558, 487, 511, 521, 532, 557, 528, 574, 526, 528, 511, 492, 476, 506, 552, 525, 541, 466, 520, 516, 475, 469, 509, 488, 522, 524, 524, 481, 520, 464, 516, 525, 543, 531, 501, 522, 506, 502, 514, 483, 535, 535, 525, 522, 517, 525, 489, 514, 503, 509, 543, 500, 494, 518, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[512, 548, 496, 491, 521, 547, 475, 531, 531, 521, 527, 479, 497, 535, 501, 503, 542, 469, 511, 519, 514, 564, 514, 532, 478, 493, 501, 498, 504, 497, 521, 524, 509, 532, 509, 507, 511, 485, 522, 532, 456, 532, 513, 529, 521, 523, 495, 481, 516, 507, 498, 509, 515, 576, 505, 526, 519, 515, 507, 482, 487, 506, 509, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[497, 488, 532, 478, 488, 500, 545, 492, 506, 477, 493, 524, 551, 514, 541, 510, 480, 514, 501, 547, 505, 485, 496, 487, 539, 528, 517, 525, 498, 501, 515, 489, 519, 488, 526, 505, 524, 515, 517, 499, 528, 490, 553, 514, 483, 530, 515, 489, 502, 527, 513, 509, 509, 533, 531, 522, 543, 520, 535, 541, 499, 529, 521, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[515, 549, 478, 527, 485, 500, 519, 513, 522, 503, 540, 506, 513, 509, 525, 490, 554, 535, 512, 504, 503, 545, 506, 537, 518, 485, 516, 515, 510, 497, 488, 504, 508, 500, 492, 523, 499, 503, 526, 506, 510, 520, 497, 472, 523, 514, 522, 506, 506, 502, 507, 564, 539, 535, 490, 520, 546, 498, 492, 538, 479, 498, 505, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 523, 489, 487, 552, 515, 526, 483, 43, 0, 0, 0, 0, 0, 0, 0, 496, 525, 506, 507, 513, 503, 502, 529, 0, 0, 0, 0, 0, 0, 0, 0, 510, 550, 499, 545, 525, 525, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 531, 480, 496, 492, 533, 461, 507, 526, 23, 18, 0, 0, 0, 0, 0, 0, 555, 487, 525, 474, 510, 518, 487, 505, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 494, 477, 525, 542, 494, 535, 29, 0, 0, 0, 0, 0, 0, 0, 518, 538, 507, 479, 480, 513, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 538, 513, 513, 521, 514, 492, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4081, 4179, 4067, 4061, 4085, 4030, 4140] avg=4096.0000 max=4179 max_rank=2 min=4030 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 482, 527, 520, 489, 493, 508, 500, 42, 0, 0, 0, 0, 0, 0, 0, 500, 497, 524, 561, 526, 508, 521, 549, 0, 0, 0, 0, 0, 0, 0, 0, 489, 523, 528, 495, 505, 477, 481, 493, 0, 0, 0, 0, 0, 0, 0, 0, 483, 465, 542, 553, 522, 474, 517, 508, 23, 17, 0, 0, 0, 0, 0, 0, 531, 550, 513, 517, 548, 534, 463, 544, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 524, 496, 490, 478, 472, 481, 27, 0, 0, 0, 0, 0, 0, 0, 534, 535, 518, 517, 477, 523, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 513, 479, 509, 534, 465, 519, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4186, 3991, 4104, 4200, 3981, 4147, 4074] avg=4096.0000 max=4200 max_rank=4 min=3981 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[551, 477, 551, 523, 490, 503, 503, 475, 41, 0, 0, 0, 0, 0, 0, 0, 523, 489, 514, 478, 522, 503, 483, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 506, 485, 523, 511, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 508, 503, 517, 520, 527, 481, 22, 16, 0, 0, 0, 0, 0, 0, 517, 517, 498, 483, 539, 548, 517, 478, 0, 0, 0, 0, 0, 0, 0, 0, 528, 512, 488, 489, 508, 543, 518, 515, 26, 0, 0, 0, 0, 0, 0, 0, 563, 555, 551, 533, 462, 497, 510, 491, 0, 0, 0, 0, 0, 0, 0, 0, 506, 486, 509, 511, 460, 526, 507, 496, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4037, 4096, 4111, 4097, 4127, 4162, 4024] avg=4096.0000 max=4162 max_rank=6 min=4024 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 548, 496, 491, 521, 547, 475, 531, 40, 0, 0, 0, 0, 0, 0, 0, 531, 521, 527, 479, 497, 535, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 542, 469, 511, 477, 514, 564, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 478, 493, 501, 498, 504, 497, 521, 524, 22, 16, 0, 0, 0, 0, 0, 0, 509, 532, 509, 507, 511, 485, 522, 532, 0, 0, 0, 0, 0, 0, 0, 0, 456, 532, 513, 529, 521, 523, 495, 481, 26, 0, 0, 0, 0, 0, 0, 0, 516, 507, 498, 509, 453, 576, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 507, 482, 487, 506, 509, 538, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4071, 4123, 4054, 4107, 4076, 4090, 4086] avg=4096.0000 max=4161 max_rank=0 min=4054 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 510, 472, 491, 492, 498, 492, 491, 46, 0, 0, 0, 0, 0, 0, 0, 519, 558, 487, 511, 521, 532, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 574, 526, 528, 464, 492, 476, 506, 552, 0, 0, 0, 0, 0, 0, 0, 0, 525, 541, 466, 520, 516, 475, 469, 509, 26, 19, 0, 0, 0, 0, 0, 0, 488, 522, 524, 524, 481, 513, 464, 516, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 531, 501, 522, 506, 502, 514, 28, 3, 0, 0, 0, 0, 0, 0, 483, 535, 535, 525, 450, 517, 525, 489, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 509, 543, 500, 494, 518, 513, 30, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4180, 4118, 4066, 4032, 4175, 4059, 4131] avg=4096.0000 max=4180 max_rank=1 min=4007 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[523, 483, 482, 523, 552, 451, 527, 505, 42, 0, 0, 0, 0, 0, 0, 0, 531, 525, 509, 503, 497, 490, 489, 494, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 501, 497, 536, 568, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 501, 530, 468, 500, 535, 516, 23, 17, 0, 0, 0, 0, 0, 0, 498, 511, 504, 490, 519, 530, 533, 536, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 514, 497, 489, 501, 525, 513, 27, 0, 0, 0, 0, 0, 0, 0, 475, 518, 540, 541, 474, 508, 555, 486, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 499, 538, 470, 502, 490, 502, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4038, 4134, 4145, 4121, 4106, 4097, 4039] avg=4096.0000 max=4145 max_rank=3 min=4038 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[497, 488, 532, 478, 488, 500, 545, 492, 40, 0, 0, 0, 0, 0, 0, 0, 506, 477, 493, 524, 551, 514, 516, 510, 0, 0, 0, 0, 0, 0, 0, 0, 480, 514, 501, 503, 505, 485, 496, 487, 0, 0, 0, 0, 0, 0, 0, 0, 539, 528, 517, 525, 498, 501, 515, 489, 21, 17, 0, 0, 0, 0, 0, 0, 519, 488, 526, 505, 524, 515, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 553, 514, 483, 530, 515, 489, 27, 0, 0, 0, 0, 0, 0, 0, 502, 527, 513, 509, 448, 533, 531, 522, 0, 0, 0, 0, 0, 0, 0, 0, 543, 520, 535, 541, 499, 529, 521, 476, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4091, 3971, 4150, 4093, 4129, 4085, 4189] avg=4096.0000 max=4189 max_rank=7 min=3971 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 549, 478, 527, 485, 500, 519, 513, 42, 0, 0, 0, 0, 0, 0, 0, 522, 503, 540, 506, 513, 509, 501, 490, 0, 0, 0, 0, 0, 0, 0, 0, 554, 535, 512, 464, 503, 545, 506, 537, 0, 0, 0, 0, 0, 0, 0, 0, 518, 485, 516, 515, 510, 497, 488, 504, 23, 15, 0, 0, 0, 0, 0, 0, 508, 500, 492, 523, 499, 503, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 497, 472, 523, 514, 522, 506, 25, 0, 0, 0, 0, 0, 0, 0, 506, 502, 507, 564, 474, 535, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0, 546, 498, 492, 538, 479, 498, 505, 505, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4084, 4156, 4071, 4057, 4089, 4098, 4085] avg=4096.0000 max=4156 max_rank=2 min=4057 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 3840, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[509, 487, 509, 544, 488, 527, 480, 505, 564, 496, 526, 513, 499, 534, 511, 539, 506, 479, 524, 513, 545, 534, 485, 533, 521, 519, 487, 487, 544, 506, 522, 512, 546, 484, 496, 495, 474, 523, 525, 508, 527, 496, 545, 510, 519, 462, 478, 556, 529, 517, 461, 517, 500, 516, 570, 546, 472, 459, 506, 545, 480, 545, 507, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[485, 482, 533, 485, 509, 471, 433, 507, 496, 539, 526, 526, 511, 477, 563, 486, 528, 499, 490, 520, 510, 546, 538, 476, 503, 524, 521, 552, 536, 520, 474, 513, 539, 522, 490, 511, 492, 559, 523, 513, 526, 494, 488, 489, 495, 485, 516, 508, 516, 518, 537, 539, 491, 540, 519, 534, 565, 514, 495, 500, 501, 523, 539, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[497, 532, 511, 526, 557, 525, 510, 513, 569, 492, 513, 478, 515, 488, 495, 502, 520, 523, 539, 504, 519, 509, 526, 479, 538, 487, 499, 481, 521, 485, 521, 502, 513, 494, 506, 542, 511, 484, 514, 552, 519, 532, 465, 515, 484, 525, 510, 484, 499, 531, 508, 498, 518, 505, 500, 537, 526, 514, 501, 562, 511, 492, 514, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[508, 485, 523, 519, 536, 516, 505, 492, 475, 516, 505, 512, 517, 463, 504, 518, 493, 534, 533, 508, 489, 504, 517, 540, 488, 512, 511, 532, 558, 537, 543, 513, 476, 557, 525, 489, 525, 489, 506, 521, 493, 486, 502, 513, 521, 484, 505, 515, 516, 532, 516, 492, 501, 508, 527, 531, 522, 511, 491, 494, 540, 542, 557, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[493, 511, 502, 487, 470, 518, 522, 517, 539, 513, 492, 511, 526, 534, 498, 498, 524, 482, 510, 514, 517, 516, 499, 470, 522, 519, 463, 488, 518, 506, 532, 539, 489, 532, 512, 523, 509, 519, 508, 521, 494, 541, 502, 530, 527, 511, 491, 503, 513, 563, 488, 553, 503, 530, 481, 542, 505, 519, 544, 517, 505, 514, 523, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[519, 490, 526, 504, 503, 519, 513, 490, 513, 500, 522, 539, 488, 518, 503, 502, 515, 536, 517, 527, 513, 527, 529, 522, 468, 491, 549, 495, 520, 497, 536, 494, 553, 503, 518, 518, 505, 533, 491, 515, 510, 497, 493, 535, 549, 538, 475, 507, 530, 512, 546, 524, 545, 504, 519, 487, 478, 523, 499, 498, 524, 474, 499, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[569, 535, 471, 514, 495, 538, 487, 519, 506, 512, 523, 514, 520, 477, 512, 459, 522, 527, 518, 504, 523, 507, 484, 491, 511, 529, 535, 518, 514, 498, 531, 505, 517, 515, 503, 495, 537, 469, 496, 450, 524, 478, 488, 499, 532, 520, 534, 515, 524, 518, 554, 511, 527, 519, 569, 505, 513, 516, 513, 520, 547, 487, 498, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[510, 544, 534, 499, 534, 518, 521, 537, 538, 520, 525, 488, 464, 509, 519, 505, 501, 486, 524, 529, 492, 525, 516, 520, 523, 559, 478, 514, 536, 468, 486, 518, 517, 533, 500, 510, 478, 546, 484, 527, 516, 528, 469, 524, 540, 497, 534, 532, 481, 536, 453, 482, 486, 561, 530, 486, 499, 495, 536, 488, 537, 502, 502, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 482, 533, 485, 509, 471, 433, 507, 18, 0, 0, 0, 0, 0, 0, 0, 496, 539, 526, 526, 511, 477, 563, 486, 8, 5, 0, 0, 0, 0, 0, 0, 528, 499, 490, 520, 510, 537, 538, 476, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 521, 552, 529, 520, 474, 513, 0, 0, 0, 0, 0, 0, 0, 0, 539, 522, 490, 511, 492, 559, 523, 513, 3, 1, 0, 0, 0, 0, 0, 0, 526, 494, 488, 489, 495, 485, 516, 508, 23, 0, 0, 0, 0, 0, 0, 0, 516, 472, 537, 539, 491, 540, 519, 534, 0, 0, 0, 0, 0, 0, 0, 0, 565, 514, 495, 500, 501, 523, 539, 508, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3923, 4137, 4098, 4136, 4153, 4024, 4148, 4149] avg=4096.0000 max=4153 max_rank=4 min=3923 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[508, 485, 523, 519, 536, 516, 505, 492, 18, 0, 0, 0, 0, 0, 0, 0, 475, 516, 505, 512, 517, 463, 504, 518, 8, 5, 0, 0, 0, 0, 0, 0, 493, 534, 533, 508, 489, 495, 517, 540, 0, 0, 0, 0, 0, 0, 0, 0, 488, 512, 511, 532, 551, 537, 543, 513, 0, 0, 0, 0, 0, 0, 0, 0, 476, 557, 525, 489, 525, 489, 506, 521, 3, 1, 0, 0, 0, 0, 0, 0, 493, 486, 502, 513, 521, 484, 505, 515, 23, 0, 0, 0, 0, 0, 0, 0, 516, 486, 516, 492, 501, 508, 527, 531, 0, 0, 0, 0, 0, 0, 0, 0, 522, 511, 491, 494, 540, 542, 557, 475, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4023, 4109, 4187, 4092, 4042, 4077, 4136] avg=4096.0000 max=4187 max_rank=3 min=4023 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 487, 509, 544, 488, 527, 480, 505, 18, 0, 0, 0, 0, 0, 0, 0, 564, 496, 526, 513, 499, 534, 511, 539, 8, 5, 0, 0, 0, 0, 0, 0, 506, 479, 524, 513, 545, 525, 485, 533, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 487, 487, 537, 506, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 546, 484, 496, 495, 474, 523, 525, 508, 3, 1, 0, 0, 0, 0, 0, 0, 527, 496, 545, 510, 519, 462, 478, 556, 22, 0, 0, 0, 0, 0, 0, 0, 529, 472, 461, 517, 500, 516, 570, 546, 0, 0, 0, 0, 0, 0, 0, 0, 472, 459, 506, 545, 480, 545, 507, 506, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4195, 4110, 4091, 4055, 4115, 4111, 4024] avg=4096.0000 max=4195 max_rank=1 min=4024 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[497, 532, 511, 526, 557, 525, 510, 513, 18, 0, 0, 0, 0, 0, 0, 0, 569, 492, 513, 478, 515, 488, 495, 502, 8, 5, 0, 0, 0, 0, 0, 0, 520, 523, 539, 504, 519, 500, 526, 479, 0, 0, 0, 0, 0, 0, 0, 0, 538, 487, 499, 481, 515, 485, 521, 502, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 506, 542, 511, 484, 514, 552, 2, 1, 0, 0, 0, 0, 0, 0, 519, 532, 465, 515, 484, 525, 510, 484, 23, 0, 0, 0, 0, 0, 0, 0, 499, 485, 508, 498, 518, 505, 500, 537, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 501, 562, 511, 492, 514, 526, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4065, 4110, 4028, 4119, 4057, 4050, 4150] avg=4096.0000 max=4189 max_rank=0 min=4028 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 511, 502, 487, 470, 518, 522, 517, 19, 0, 0, 0, 0, 0, 0, 0, 539, 513, 492, 511, 526, 534, 498, 498, 8, 5, 0, 0, 0, 0, 0, 0, 524, 482, 510, 514, 517, 507, 499, 470, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 463, 488, 512, 506, 532, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 532, 512, 523, 509, 519, 508, 521, 2, 1, 0, 0, 0, 0, 0, 0, 494, 541, 502, 530, 527, 511, 491, 503, 25, 0, 0, 0, 0, 0, 0, 0, 513, 514, 488, 553, 503, 530, 481, 542, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 544, 517, 505, 514, 523, 506, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4124, 4023, 4081, 4116, 4124, 4124, 4137] avg=4096.0000 max=4137 max_rank=7 min=4023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 490, 526, 504, 503, 519, 513, 490, 18, 0, 0, 0, 0, 0, 0, 0, 513, 500, 522, 539, 488, 518, 503, 502, 8, 5, 0, 0, 0, 0, 0, 0, 515, 536, 517, 527, 513, 518, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 468, 491, 549, 495, 514, 497, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 553, 503, 518, 518, 505, 533, 491, 515, 2, 1, 0, 0, 0, 0, 0, 0, 510, 497, 493, 535, 549, 538, 475, 507, 22, 0, 0, 0, 0, 0, 0, 0, 530, 467, 546, 524, 545, 504, 519, 487, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 499, 498, 524, 474, 499, 481, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4098, 4177, 4044, 4139, 4126, 4122, 3980] avg=4096.0000 max=4177 max_rank=2 min=3980 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[569, 535, 471, 514, 495, 538, 487, 519, 22, 0, 0, 0, 0, 0, 0, 0, 506, 512, 523, 514, 520, 477, 512, 459, 12, 10, 0, 0, 0, 0, 0, 0, 522, 527, 518, 504, 523, 488, 484, 491, 0, 0, 0, 0, 0, 0, 0, 0, 511, 529, 535, 518, 499, 498, 531, 505, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 503, 495, 537, 469, 496, 450, 6, 7, 0, 0, 0, 0, 0, 0, 524, 478, 488, 499, 532, 520, 534, 515, 27, 0, 0, 0, 0, 0, 0, 0, 524, 459, 554, 511, 527, 519, 569, 505, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 513, 520, 547, 487, 498, 507, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4045, 4057, 4126, 3995, 4117, 4168, 4110] avg=4096.0000 max=4168 max_rank=6 min=3995 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 544, 534, 499, 534, 518, 521, 537, 19, 0, 0, 0, 0, 0, 0, 0, 538, 520, 525, 488, 464, 509, 519, 505, 8, 5, 0, 0, 0, 0, 0, 0, 501, 486, 524, 529, 492, 516, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 523, 559, 478, 514, 529, 468, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 500, 510, 478, 546, 484, 527, 3, 1, 0, 0, 0, 0, 0, 0, 516, 528, 469, 524, 540, 497, 534, 532, 23, 0, 0, 0, 0, 0, 0, 0, 481, 489, 453, 482, 486, 561, 530, 486, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 536, 488, 537, 502, 502, 519, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4216, 4081, 4084, 4075, 4099, 4163, 3968, 4082] avg=4096.0000 max=4216 max_rank=0 min=3968 min_rank=6 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[537, 473, 536, 505, 513, 512, 483, 500, 526, 522, 535, 545, 505, 513, 495, 538, 518, 505, 509, 504, 521, 523, 481, 553, 494, 508, 518, 482, 462, 487, 500, 506, 496, 531, 518, 523, 516, 530, 528, 487, 513, 506, 513, 494, 527, 491, 533, 536, 508, 523, 521, 491, 507, 539, 520, 541, 472, 542, 480, 551, 490, 524, 501, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[479, 524, 541, 536, 489, 525, 541, 521, 538, 489, 507, 556, 518, 523, 500, 510, 478, 497, 513, 535, 538, 540, 527, 499, 491, 468, 518, 537, 493, 496, 529, 490, 478, 508, 493, 510, 490, 523, 512, 521, 508, 518, 534, 466, 468, 526, 501, 507, 483, 519, 545, 477, 479, 516, 507, 538, 484, 537, 528, 561, 490, 536, 550, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[490, 500, 518, 508, 503, 515, 499, 529, 540, 537, 500, 541, 541, 529, 536, 518, 513, 495, 521, 514, 513, 581, 497, 518, 532, 501, 486, 495, 502, 537, 513, 525, 479, 480, 499, 517, 532, 509, 498, 485, 495, 514, 549, 463, 472, 535, 503, 487, 490, 536, 535, 479, 510, 524, 481, 516, 485, 504, 520, 526, 526, 487, 559, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[518, 510, 530, 498, 512, 530, 521, 504, 504, 507, 520, 531, 537, 529, 515, 489, 529, 528, 515, 508, 491, 502, 514, 498, 497, 483, 518, 536, 494, 509, 470, 500, 478, 498, 509, 489, 518, 530, 510, 528, 561, 515, 528, 522, 497, 502, 496, 507, 506, 472, 511, 536, 499, 516, 501, 530, 539, 566, 510, 514, 507, 501, 519, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[515, 525, 511, 535, 518, 487, 526, 511, 558, 513, 507, 512, 536, 466, 501, 493, 510, 487, 521, 510, 535, 527, 556, 508, 496, 531, 532, 514, 538, 538, 519, 541, 453, 517, 510, 503, 494, 494, 469, 521, 499, 500, 522, 483, 493, 496, 542, 475, 518, 519, 525, 549, 529, 525, 515, 478, 506, 514, 516, 472, 513, 504, 471, 566] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[520, 507, 518, 512, 520, 552, 503, 509, 509, 517, 512, 533, 498, 506, 534, 502, 533, 493, 526, 515, 500, 509, 525, 514, 496, 499, 514, 498, 506, 520, 519, 526, 500, 521, 515, 492, 518, 504, 512, 527, 482, 493, 520, 511, 545, 472, 547, 532, 475, 480, 480, 540, 509, 544, 526, 530, 527, 509, 502, 480, 526, 509, 518, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[521, 454, 529, 509, 503, 525, 556, 523, 492, 521, 561, 486, 496, 508, 517, 489, 499, 547, 527, 521, 514, 522, 473, 515, 521, 527, 471, 513, 543, 531, 489, 494, 510, 485, 489, 545, 503, 525, 497, 502, 501, 513, 527, 562, 492, 574, 525, 511, 510, 495, 518, 503, 529, 489, 511, 521, 505, 506, 536, 528, 465, 501, 516, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[532, 497, 543, 512, 460, 511, 536, 510, 523, 536, 486, 514, 501, 515, 514, 515, 546, 533, 550, 532, 495, 517, 515, 515, 530, 515, 485, 509, 490, 531, 500, 503, 476, 507, 527, 513, 513, 558, 510, 508, 538, 504, 518, 536, 501, 501, 494, 512, 491, 522, 545, 479, 488, 518, 509, 518, 485, 514, 501, 494, 491, 529, 467, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[479, 524, 526, 536, 489, 525, 541, 521, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 507, 518, 518, 523, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 478, 497, 513, 535, 538, 502, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0, 491, 468, 518, 537, 493, 496, 529, 490, 33, 0, 0, 0, 0, 0, 0, 0, 478, 508, 493, 510, 490, 523, 512, 521, 38, 3, 0, 0, 0, 0, 0, 0, 508, 518, 534, 466, 468, 526, 501, 507, 5, 2, 0, 0, 0, 0, 0, 0, 483, 519, 545, 477, 479, 516, 507, 538, 3, 0, 0, 0, 0, 0, 0, 0, 484, 537, 528, 561, 490, 536, 550, 509, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4103, 4089, 4055, 4076, 4035, 4067, 4202] avg=4096.0000 max=4202 max_rank=7 min=4035 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[520, 507, 505, 512, 520, 552, 503, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 512, 497, 498, 506, 534, 502, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 526, 515, 500, 473, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 499, 514, 498, 506, 520, 519, 526, 31, 0, 0, 0, 0, 0, 0, 0, 500, 521, 515, 492, 518, 504, 512, 527, 36, 3, 0, 0, 0, 0, 0, 0, 482, 493, 520, 511, 545, 472, 547, 532, 5, 2, 0, 0, 0, 0, 0, 0, 475, 480, 480, 540, 509, 544, 526, 530, 2, 0, 0, 0, 0, 0, 0, 0, 527, 509, 502, 480, 526, 509, 518, 477, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4075, 4079, 4109, 4128, 4109, 4086, 4054] avg=4096.0000 max=4128 max_rank=0 min=4054 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[537, 473, 521, 505, 513, 512, 483, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 535, 508, 505, 513, 495, 538, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 509, 504, 521, 486, 481, 553, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 518, 482, 462, 487, 500, 506, 32, 0, 0, 0, 0, 0, 0, 0, 496, 531, 518, 523, 516, 530, 528, 487, 37, 3, 0, 0, 0, 0, 0, 0, 513, 506, 513, 494, 527, 491, 533, 536, 5, 2, 0, 0, 0, 0, 0, 0, 508, 523, 521, 491, 507, 539, 520, 541, 3, 0, 0, 0, 0, 0, 0, 0, 472, 542, 480, 551, 490, 524, 501, 507, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4142, 4077, 3989, 4169, 4120, 4153, 4074] avg=4096.0000 max=4169 max_rank=4 min=3989 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 510, 515, 498, 512, 530, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 520, 495, 537, 529, 515, 489, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 515, 508, 491, 467, 514, 498, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 518, 536, 494, 509, 470, 500, 31, 0, 0, 0, 0, 0, 0, 0, 478, 498, 509, 489, 518, 530, 510, 528, 35, 3, 0, 0, 0, 0, 0, 0, 561, 515, 528, 522, 497, 502, 496, 507, 5, 2, 0, 0, 0, 0, 0, 0, 506, 472, 511, 536, 499, 516, 501, 530, 3, 0, 0, 0, 0, 0, 0, 0, 539, 566, 510, 514, 507, 501, 519, 506, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4050, 4038, 4098, 4135, 4074, 4169] avg=4096.0000 max=4169 max_rank=7 min=4038 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[490, 500, 505, 508, 503, 515, 499, 529, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 500, 504, 541, 529, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 521, 514, 513, 540, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 486, 495, 502, 537, 513, 525, 32, 0, 0, 0, 0, 0, 0, 0, 479, 480, 499, 517, 532, 509, 498, 485, 41, 3, 0, 0, 0, 0, 0, 0, 495, 514, 549, 463, 472, 535, 503, 487, 5, 2, 0, 0, 0, 0, 0, 0, 490, 536, 535, 479, 510, 524, 481, 516, 2, 0, 0, 0, 0, 0, 0, 0, 485, 504, 520, 526, 526, 487, 559, 526, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4205, 4111, 4123, 4043, 4025, 4073, 4139] avg=4096.0000 max=4205 max_rank=1 min=4025 min_rank=5 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 525, 498, 535, 518, 487, 526, 511, 0, 0, 0, 0, 0, 0, 0, 0, 558, 513, 507, 477, 536, 466, 501, 493, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 521, 510, 535, 490, 556, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 532, 514, 538, 538, 519, 541, 30, 0, 0, 0, 0, 0, 0, 0, 453, 517, 510, 503, 494, 494, 469, 521, 37, 3, 0, 0, 0, 0, 0, 0, 499, 500, 522, 483, 493, 496, 542, 475, 5, 2, 0, 0, 0, 0, 0, 0, 518, 519, 525, 549, 529, 525, 515, 478, 2, 0, 0, 0, 0, 0, 0, 0, 506, 514, 516, 472, 513, 504, 471, 566, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4051, 4117, 4239, 4001, 4017, 4160, 4068] avg=4096.0000 max=4239 max_rank=3 min=4001 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 454, 506, 509, 503, 525, 556, 523, 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, 561, 443, 496, 508, 517, 489, 0, 0, 0, 0, 0, 0, 0, 0, 499, 547, 527, 521, 514, 482, 473, 515, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 471, 513, 543, 531, 489, 494, 33, 0, 0, 0, 0, 0, 0, 0, 510, 485, 489, 545, 503, 525, 497, 502, 40, 6, 0, 0, 0, 0, 0, 0, 501, 513, 527, 562, 492, 574, 525, 511, 7, 4, 0, 0, 0, 0, 0, 0, 510, 495, 518, 503, 529, 489, 511, 521, 6, 0, 0, 0, 0, 0, 0, 0, 505, 506, 536, 528, 465, 501, 516, 477, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4027, 4078, 4122, 4102, 4216, 4082, 4044] avg=4096.0000 max=4216 max_rank=5 min=4027 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[532, 497, 528, 512, 460, 511, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 523, 536, 486, 479, 501, 515, 514, 515, 0, 0, 0, 0, 0, 0, 0, 0, 546, 533, 550, 532, 495, 481, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 530, 515, 485, 509, 490, 531, 500, 503, 30, 0, 0, 0, 0, 0, 0, 0, 476, 507, 527, 513, 513, 558, 510, 508, 36, 3, 0, 0, 0, 0, 0, 0, 538, 504, 518, 536, 501, 501, 494, 512, 5, 2, 0, 0, 0, 0, 0, 0, 491, 522, 545, 479, 488, 518, 509, 518, 3, 0, 0, 0, 0, 0, 0, 0, 485, 514, 501, 494, 491, 529, 467, 530, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4069, 4167, 4093, 4151, 4111, 4073, 4018] avg=4096.0000 max=4167 max_rank=2 min=4018 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[515, 521, 509, 516, 514, 514, 506, 522, 502, 500, 527, 512, 530, 517, 501, 479, 500, 529, 486, 487, 482, 486, 475, 509, 536, 520, 546, 527, 472, 537, 497, 507, 491, 480, 519, 504, 524, 489, 490, 503, 539, 497, 489, 523, 531, 511, 526, 521, 513, 562, 515, 532, 538, 539, 523, 507, 543, 512, 488, 473, 522, 519, 531, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[500, 542, 502, 518, 503, 539, 507, 529, 527, 517, 527, 534, 497, 532, 532, 510, 518, 465, 494, 509, 497, 471, 515, 509, 517, 481, 527, 533, 506, 545, 525, 510, 523, 524, 529, 535, 511, 497, 515, 520, 507, 495, 514, 511, 531, 528, 500, 496, 516, 508, 547, 487, 501, 486, 510, 484, 509, 516, 502, 516, 527, 508, 490, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[485, 507, 506, 524, 507, 515, 506, 503, 498, 577, 495, 491, 502, 527, 551, 532, 498, 520, 525, 505, 519, 486, 537, 511, 525, 529, 502, 506, 493, 525, 491, 505, 519, 509, 524, 461, 564, 530, 492, 474, 506, 524, 518, 530, 525, 526, 525, 535, 494, 492, 494, 523, 513, 512, 473, 506, 540, 505, 504, 504, 514, 502, 503, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[510, 490, 502, 528, 520, 481, 541, 499, 517, 522, 537, 497, 513, 521, 514, 508, 495, 491, 510, 491, 508, 497, 496, 491, 500, 446, 480, 514, 527, 545, 490, 512, 568, 506, 547, 537, 460, 535, 518, 529, 511, 498, 505, 529, 501, 523, 563, 521, 516, 511, 529, 509, 520, 487, 529, 493, 545, 527, 523, 501, 513, 486, 519, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[515, 493, 510, 498, 507, 540, 477, 519, 514, 497, 509, 508, 490, 526, 524, 502, 505, 503, 525, 499, 513, 525, 515, 523, 513, 495, 523, 494, 541, 536, 495, 526, 509, 476, 521, 538, 529, 503, 482, 547, 565, 490, 519, 554, 514, 506, 514, 511, 480, 512, 507, 526, 486, 516, 514, 532, 556, 483, 516, 469, 520, 475, 518, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[528, 508, 514, 507, 513, 499, 509, 482, 526, 509, 500, 493, 512, 530, 495, 456, 494, 493, 532, 525, 510, 482, 485, 550, 471, 509, 496, 525, 499, 530, 500, 521, 511, 533, 484, 548, 573, 502, 551, 500, 540, 529, 509, 537, 531, 483, 462, 546, 510, 509, 495, 546, 499, 553, 525, 511, 530, 496, 527, 496, 480, 527, 517, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[554, 537, 503, 552, 510, 501, 523, 518, 504, 527, 480, 500, 479, 506, 518, 513, 526, 440, 535, 492, 489, 518, 529, 522, 521, 514, 510, 525, 530, 497, 526, 524, 490, 514, 513, 554, 494, 500, 500, 519, 496, 519, 466, 510, 518, 558, 507, 531, 480, 541, 533, 558, 475, 524, 520, 521, 513, 521, 485, 493, 503, 496, 486, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[517, 522, 545, 535, 518, 487, 522, 524, 507, 538, 477, 498, 463, 508, 539, 484, 514, 561, 545, 537, 499, 523, 506, 480, 511, 477, 481, 494, 514, 466, 555, 501, 533, 499, 517, 508, 489, 500, 484, 485, 563, 513, 508, 493, 498, 488, 558, 550, 494, 490, 523, 531, 530, 525, 566, 508, 492, 502, 479, 501, 513, 519, 509, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[515, 521, 509, 505, 514, 514, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 527, 512, 530, 517, 501, 479, 11, 0, 0, 0, 0, 0, 0, 0, 500, 529, 486, 487, 482, 486, 475, 509, 46, 0, 0, 0, 0, 0, 0, 0, 536, 520, 546, 527, 472, 537, 497, 507, 7, 0, 0, 0, 0, 0, 0, 0, 491, 480, 519, 497, 524, 489, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 491, 497, 489, 523, 531, 511, 526, 521, 0, 0, 0, 0, 0, 0, 0, 0, 513, 562, 515, 515, 538, 539, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 543, 512, 488, 473, 522, 519, 531, 533, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4079, 4000, 4149, 3993, 4089, 4212, 4140] avg=4096.0000 max=4212 max_rank=6 min=3993 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[515, 493, 510, 488, 507, 540, 477, 519, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 509, 508, 490, 526, 524, 502, 10, 0, 0, 0, 0, 0, 0, 0, 505, 503, 525, 499, 513, 525, 515, 523, 48, 0, 0, 0, 0, 0, 0, 0, 513, 495, 523, 494, 541, 536, 495, 526, 8, 0, 0, 0, 0, 0, 0, 0, 509, 476, 521, 530, 529, 503, 482, 547, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 519, 554, 514, 506, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 480, 512, 507, 509, 486, 516, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 556, 483, 516, 469, 520, 475, 518, 520, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4080, 4156, 4131, 4097, 4123, 4056, 4076] avg=4096.0000 max=4156 max_rank=2 min=4049 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[510, 490, 502, 517, 520, 481, 541, 499, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 537, 497, 513, 521, 514, 508, 11, 0, 0, 0, 0, 0, 0, 0, 495, 491, 510, 491, 508, 497, 496, 491, 43, 0, 0, 0, 0, 0, 0, 0, 500, 446, 480, 514, 527, 545, 490, 512, 8, 0, 0, 0, 0, 0, 0, 0, 568, 506, 547, 529, 460, 535, 518, 529, 0, 0, 0, 0, 0, 0, 0, 0, 467, 498, 505, 529, 501, 523, 563, 521, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 529, 492, 520, 487, 529, 493, 0, 0, 0, 0, 0, 0, 0, 0, 545, 527, 523, 501, 513, 486, 519, 516, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4140, 4022, 4022, 4192, 4107, 4077, 4148] avg=4096.0000 max=4192 max_rank=4 min=4022 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 542, 502, 507, 503, 539, 507, 529, 0, 0, 0, 0, 0, 0, 0, 0, 527, 517, 527, 534, 497, 532, 532, 510, 11, 0, 0, 0, 0, 0, 0, 0, 518, 465, 494, 509, 497, 471, 515, 509, 43, 0, 0, 0, 0, 0, 0, 0, 517, 481, 527, 533, 506, 545, 525, 510, 8, 0, 0, 0, 0, 0, 0, 0, 523, 524, 529, 527, 511, 497, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 463, 495, 514, 511, 531, 528, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 547, 471, 501, 486, 510, 484, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 502, 516, 527, 508, 490, 487, 16, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4187, 4021, 4152, 4146, 4038, 4023, 4072] avg=4096.0000 max=4187 max_rank=1 min=4021 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 522, 545, 524, 518, 487, 522, 524, 0, 0, 0, 0, 0, 0, 0, 0, 507, 538, 477, 498, 463, 508, 539, 484, 11, 0, 0, 0, 0, 0, 0, 0, 514, 561, 545, 537, 499, 523, 506, 480, 48, 0, 0, 0, 0, 0, 0, 0, 511, 477, 481, 494, 514, 466, 555, 501, 7, 0, 0, 0, 0, 0, 0, 0, 533, 499, 517, 501, 489, 500, 484, 485, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 508, 493, 498, 488, 558, 550, 0, 0, 0, 0, 0, 0, 0, 0, 494, 490, 523, 514, 530, 525, 566, 508, 0, 0, 0, 0, 0, 0, 0, 0, 492, 502, 479, 501, 513, 519, 509, 552, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4025, 4213, 4006, 4008, 4121, 4150, 4086] avg=4096.0000 max=4213 max_rank=2 min=4006 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[485, 507, 506, 513, 507, 515, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 498, 577, 495, 491, 502, 527, 551, 532, 11, 0, 0, 0, 0, 0, 0, 0, 498, 520, 525, 505, 519, 486, 537, 511, 43, 0, 0, 0, 0, 0, 0, 0, 525, 529, 502, 506, 493, 525, 491, 505, 7, 0, 0, 0, 0, 0, 0, 0, 519, 509, 524, 454, 564, 530, 492, 474, 0, 0, 0, 0, 0, 0, 0, 0, 462, 524, 518, 530, 525, 526, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 494, 506, 513, 512, 473, 506, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 504, 504, 514, 502, 503, 524, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4184, 4144, 4083, 4066, 4145, 3990, 4114] avg=4096.0000 max=4184 max_rank=1 min=3990 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 508, 514, 483, 513, 499, 509, 482, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 500, 493, 512, 530, 495, 456, 14, 0, 0, 0, 0, 0, 0, 0, 494, 493, 532, 525, 510, 482, 485, 550, 48, 0, 0, 0, 0, 0, 0, 0, 471, 509, 496, 525, 499, 530, 500, 521, 12, 7, 0, 0, 0, 0, 0, 0, 511, 533, 484, 536, 573, 502, 551, 500, 0, 0, 0, 0, 0, 0, 0, 0, 486, 529, 509, 537, 531, 483, 462, 546, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 495, 524, 499, 553, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 496, 527, 496, 480, 527, 517, 505, 22, 6, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4035, 4119, 4070, 4190, 4083, 4126, 4109] avg=4096.0000 max=4190 max_rank=4 min=4035 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[554, 537, 503, 541, 510, 501, 523, 518, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 480, 500, 479, 506, 518, 513, 11, 0, 0, 0, 0, 0, 0, 0, 526, 440, 535, 492, 489, 518, 529, 522, 42, 0, 0, 0, 0, 0, 0, 0, 521, 514, 510, 525, 530, 497, 526, 524, 8, 0, 0, 0, 0, 0, 0, 0, 490, 514, 513, 546, 494, 500, 500, 519, 0, 0, 0, 0, 0, 0, 0, 0, 453, 519, 466, 510, 518, 558, 507, 531, 0, 0, 0, 0, 0, 0, 0, 0, 480, 541, 533, 540, 475, 524, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 485, 493, 503, 496, 486, 507, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4038, 4093, 4155, 4076, 4062, 4134, 4023] avg=4096.0000 max=4187 max_rank=0 min=4023 min_rank=7 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[508, 509, 528, 534, 502, 525, 522, 530, 476, 516, 515, 508, 520, 495, 527, 524, 498, 515, 507, 490, 507, 491, 538, 515, 495, 499, 508, 509, 513, 492, 516, 504, 534, 498, 518, 545, 559, 535, 480, 518, 534, 493, 499, 529, 512, 482, 505, 499, 538, 487, 489, 507, 535, 487, 526, 509, 505, 497, 551, 517, 504, 554, 490, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[543, 538, 529, 532, 508, 522, 520, 524, 501, 535, 496, 482, 513, 492, 489, 567, 524, 514, 519, 520, 498, 487, 522, 477, 557, 509, 511, 497, 501, 526, 532, 501, 492, 510, 510, 501, 501, 534, 518, 501, 455, 530, 527, 523, 502, 492, 493, 496, 532, 490, 540, 534, 551, 520, 506, 490, 449, 487, 535, 524, 504, 535, 520, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[513, 530, 472, 504, 549, 545, 521, 488, 551, 540, 529, 531, 448, 528, 510, 575, 497, 493, 507, 501, 483, 532, 497, 481, 542, 532, 508, 503, 526, 487, 506, 499, 513, 514, 532, 530, 521, 508, 524, 521, 498, 508, 520, 483, 513, 495, 516, 503, 500, 497, 535, 526, 523, 527, 485, 509, 514, 494, 511, 476, 529, 513, 527, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[512, 506, 531, 524, 521, 496, 560, 513, 498, 501, 456, 488, 516, 529, 483, 517, 476, 513, 561, 573, 568, 468, 527, 519, 521, 484, 529, 466, 501, 540, 517, 504, 517, 507, 548, 500, 540, 482, 501, 499, 513, 496, 483, 502, 466, 542, 519, 513, 497, 531, 525, 509, 521, 535, 511, 537, 499, 502, 490, 521, 542, 510, 494, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[506, 483, 484, 496, 481, 524, 509, 523, 486, 516, 484, 516, 528, 557, 502, 517, 525, 508, 528, 503, 525, 545, 481, 518, 523, 505, 489, 499, 499, 557, 514, 508, 521, 527, 541, 475, 492, 462, 508, 502, 519, 518, 511, 496, 524, 498, 483, 517, 552, 529, 543, 500, 492, 532, 504, 541, 525, 511, 513, 496, 546, 485, 528, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[518, 533, 476, 505, 478, 553, 550, 533, 548, 542, 517, 528, 526, 511, 510, 503, 496, 531, 480, 533, 504, 490, 518, 524, 513, 476, 497, 479, 498, 485, 528, 498, 526, 521, 515, 523, 563, 494, 518, 486, 513, 467, 512, 507, 479, 515, 496, 480, 524, 514, 541, 544, 525, 498, 550, 506, 531, 495, 489, 508, 534, 487, 505, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[544, 540, 517, 509, 507, 514, 553, 480, 551, 534, 480, 473, 522, 502, 511, 508, 525, 541, 502, 558, 529, 501, 506, 518, 492, 501, 547, 517, 512, 479, 515, 494, 506, 467, 525, 511, 520, 512, 539, 523, 462, 458, 508, 514, 493, 484, 501, 491, 477, 494, 509, 495, 533, 524, 550, 543, 528, 501, 490, 523, 525, 505, 525, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[565, 504, 500, 507, 508, 531, 485, 494, 520, 516, 513, 527, 522, 482, 571, 541, 494, 531, 510, 471, 511, 496, 501, 551, 520, 536, 517, 529, 538, 546, 494, 535, 490, 475, 500, 548, 494, 495, 516, 486, 500, 524, 509, 484, 496, 556, 529, 514, 527, 512, 534, 492, 494, 510, 538, 472, 510, 530, 502, 487, 524, 475, 483, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 530, 472, 504, 549, 545, 485, 488, 0, 0, 0, 0, 0, 0, 0, 0, 551, 540, 529, 531, 448, 528, 510, 556, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 507, 501, 483, 532, 497, 481, 0, 0, 0, 0, 0, 0, 0, 0, 542, 532, 508, 503, 526, 487, 506, 499, 8, 2, 0, 0, 0, 0, 0, 0, 513, 514, 532, 530, 519, 508, 524, 521, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 520, 483, 513, 495, 516, 503, 43, 28, 0, 0, 0, 0, 0, 0, 500, 497, 492, 526, 523, 527, 485, 509, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 511, 476, 529, 513, 527, 475, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4193, 3991, 4113, 4161, 4107, 4059, 4058] avg=4096.0000 max=4193 max_rank=1 min=3991 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 538, 529, 532, 508, 522, 484, 524, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 496, 482, 513, 492, 489, 548, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 519, 520, 498, 487, 522, 477, 0, 0, 0, 0, 0, 0, 0, 0, 557, 509, 511, 497, 501, 526, 532, 501, 8, 2, 0, 0, 0, 0, 0, 0, 492, 510, 510, 501, 499, 534, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0, 455, 530, 527, 523, 502, 492, 493, 496, 43, 28, 0, 0, 0, 0, 0, 0, 532, 490, 497, 534, 551, 520, 506, 490, 0, 0, 0, 0, 0, 0, 0, 0, 449, 487, 535, 524, 504, 535, 520, 480, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4056, 4061, 4144, 4065, 4089, 4120, 4053] avg=4096.0000 max=4180 max_rank=0 min=4053 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 533, 476, 505, 478, 553, 512, 533, 0, 0, 0, 0, 0, 0, 0, 0, 548, 542, 517, 528, 526, 511, 510, 486, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 480, 533, 504, 490, 518, 524, 0, 0, 0, 0, 0, 0, 0, 0, 513, 476, 497, 479, 498, 485, 528, 498, 7, 3, 0, 0, 0, 0, 0, 0, 526, 521, 515, 523, 560, 494, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 513, 467, 512, 507, 479, 515, 496, 480, 43, 29, 0, 0, 0, 0, 0, 0, 524, 514, 498, 544, 525, 498, 550, 506, 0, 0, 0, 0, 0, 0, 0, 0, 531, 495, 489, 508, 534, 487, 505, 521, 10, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4168, 4076, 3984, 4143, 4041, 4159, 4089] avg=4096.0000 max=4168 max_rank=1 min=3984 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[508, 509, 528, 534, 502, 525, 486, 530, 0, 0, 0, 0, 0, 0, 0, 0, 476, 516, 515, 508, 520, 495, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 507, 490, 507, 491, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 508, 509, 513, 492, 516, 504, 8, 3, 0, 0, 0, 0, 0, 0, 534, 498, 518, 545, 556, 535, 480, 518, 0, 0, 0, 0, 0, 0, 0, 0, 534, 493, 499, 529, 512, 482, 505, 499, 39, 28, 0, 0, 0, 0, 0, 0, 538, 487, 450, 507, 535, 487, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 551, 517, 504, 554, 490, 496, 10, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4063, 4061, 4047, 4184, 4120, 4039, 4132] avg=4096.0000 max=4184 max_rank=4 min=4039 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[512, 506, 531, 524, 521, 496, 521, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 501, 456, 488, 516, 529, 483, 500, 0, 0, 0, 0, 0, 0, 0, 0, 476, 513, 561, 573, 568, 468, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 521, 484, 529, 466, 501, 540, 517, 504, 7, 3, 0, 0, 0, 0, 0, 0, 517, 507, 548, 500, 537, 482, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 483, 502, 466, 542, 519, 513, 42, 30, 0, 0, 0, 0, 0, 0, 497, 531, 483, 509, 521, 535, 511, 537, 0, 0, 0, 0, 0, 0, 0, 0, 499, 502, 490, 521, 542, 510, 494, 498, 10, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 3971, 4205, 4072, 4091, 4106, 4124, 4075] avg=4096.0000 max=4205 max_rank=2 min=3971 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 483, 484, 496, 481, 524, 474, 523, 0, 0, 0, 0, 0, 0, 0, 0, 486, 516, 484, 516, 528, 557, 502, 500, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 528, 503, 525, 545, 481, 518, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 489, 499, 499, 557, 514, 508, 7, 2, 0, 0, 0, 0, 0, 0, 521, 527, 541, 475, 490, 462, 508, 502, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 511, 496, 524, 498, 483, 517, 43, 27, 0, 0, 0, 0, 0, 0, 552, 529, 500, 500, 492, 532, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 513, 496, 546, 485, 528, 538, 10, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3971, 4089, 4133, 4103, 4026, 4136, 4150, 4160] avg=4096.0000 max=4160 max_rank=7 min=3971 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[544, 540, 517, 509, 507, 514, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 551, 534, 480, 473, 522, 502, 511, 482, 0, 0, 0, 0, 0, 0, 0, 0, 525, 541, 502, 554, 529, 501, 506, 518, 0, 0, 0, 0, 0, 0, 0, 0, 492, 501, 547, 517, 512, 479, 515, 494, 12, 7, 4, 0, 0, 0, 0, 0, 506, 467, 525, 511, 513, 512, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 462, 458, 508, 514, 493, 484, 501, 491, 44, 33, 0, 0, 0, 0, 0, 0, 477, 494, 465, 495, 533, 524, 550, 543, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 490, 523, 525, 505, 525, 550, 14, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4055, 4176, 4080, 4096, 3988, 4081, 4175] avg=4096.0000 max=4176 max_rank=2 min=3988 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[565, 504, 500, 507, 508, 531, 451, 494, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 513, 527, 522, 482, 571, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 531, 510, 471, 511, 496, 501, 551, 0, 0, 0, 0, 0, 0, 0, 0, 520, 536, 517, 529, 538, 546, 494, 535, 8, 2, 0, 0, 0, 0, 0, 0, 490, 475, 500, 548, 492, 495, 516, 486, 0, 0, 0, 0, 0, 0, 0, 0, 500, 524, 509, 484, 496, 556, 529, 514, 43, 26, 0, 0, 0, 0, 0, 0, 527, 512, 491, 492, 494, 510, 538, 472, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 502, 487, 524, 475, 483, 496, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4173, 4065, 4225, 4002, 4181, 4036, 4026] avg=4096.0000 max=4225 max_rank=3 min=4002 min_rank=4 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 15:43:14] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 280.7 | throughput per GPU (TFLOP/s/GPU): 440.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.427963E+00 | loss scale: 1.0 | grad norm: 3.409 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 15:43:14 +[rank0]:[W826 15:43:15.875262413 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 15:43:17.797000 124378 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 15:43:17.797000 124378 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 124469 closing signal SIGTERM +W0826 15:43:17.798000 124378 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 124470 closing signal SIGTERM +W0826 15:43:17.798000 124378 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 124471 closing signal SIGTERM +W0826 15:43:17.798000 124378 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 124472 closing signal SIGTERM +W0826 15:43:17.799000 124378 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 124473 closing signal SIGTERM +W0826 15:43:17.799000 124378 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 124474 closing signal SIGTERM +W0826 15:43:17.799000 124378 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 124476 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 124378 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index d283634ea4f092005f88c8007a9e7d27c3809d14..179ce3433902686a392a243be5dcbb72c7d37a8d 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -6,3 +6,4 @@ QWEN3_30B echo forced 2.00 OK 133533 1 0.0 1.0 /workspace/tierb_study_20260826_1 DSV2_LITE_L24 echo natural 2.00 OK 140961 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 DSV2_LITE_L24 echo forced 2.00 OK 140541 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 QWEN2_57B echo natural 2.00 OK 109133 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN2_57B echo forced 2.00 OK 109131 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index b21bd48bae9d2b2a127fbd66722ce46bd3b8317c..9afb1b98c09ae2c363c2077deb7bb4bc4b6e0287 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -2,3 +2,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib QWEN3_235B echo SELECTED 2.00 1.0 0.0 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 QWEN3_30B echo SELECTED 2.00 1.0 0.0 133569 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 DSV2_LITE_L24 echo SELECTED 2.00 1.0 0.0 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN2_57B echo SELECTED 2.00 1.0 0.0 109133 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json index 078922113f45701185826a3f8787c7f05fecdd8b..b6ab4a5b5e33c0c78706bff1561cea2d00cd0322 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -179,8 +179,8 @@ "branch": "token-drop-analysis", "capacity": "2.00", "capacity_mode": "static", - "cold_expert_fraction_max": 0.828125, - "cold_expert_fraction_mean": 0.3279947916666667, + "cold_expert_fraction_max": 0.796875, + "cold_expert_fraction_mean": 0.33098958333333334, "cold_expert_fraction_min": 0.0, "cold_expert_fraction_n": 120, "commit": "b099fd84", @@ -200,8 +200,8 @@ "global_batch_size": "8", "global_snapshots": 120, "hottest_expert_token_fraction_max": 1.0, - "hottest_expert_token_fraction_mean": 0.8515007019042968, - "hottest_expert_token_fraction_min": 0.2647705078125, + "hottest_expert_token_fraction_mean": 0.8510676066080729, + "hottest_expert_token_fraction_min": 0.275054931640625, "hottest_expert_token_fraction_n": 120, "incomplete_snapshots": 0, "malformed_records": 0, @@ -217,23 +217,23 @@ "post_physical_rank_imax_mean": 1.0, "post_physical_rank_imax_min": 1.0, "post_physical_rank_imax_n": 120, - "pre_home_rank_cv_max": 1.2169604837256403, - "pre_home_rank_cv_mean": 0.6377553523750428, - "pre_home_rank_cv_min": 0.09633592940981593, + "pre_home_rank_cv_max": 1.2755844916699237, + "pre_home_rank_cv_mean": 0.6343602950592665, + "pre_home_rank_cv_min": 0.0020009968962783516, "pre_home_rank_cv_n": 120, - "pre_home_rank_imax_max": 3.978790283203125, - "pre_home_rank_imax_mean": 2.1783960978190104, - "pre_home_rank_imax_min": 1.12921142578125, + "pre_home_rank_imax_max": 3.97802734375, + "pre_home_rank_imax_mean": 2.162261708577474, + "pre_home_rank_imax_min": 1.004638671875, "pre_home_rank_imax_n": 120, - "raw_expert_cv_max": 2.6426244377115378, - "raw_expert_cv_mean": 1.9430158591244282, - "raw_expert_cv_min": 0.5540860535283595, + "raw_expert_cv_max": 2.643741260408719, + "raw_expert_cv_mean": 1.9491049129110607, + "raw_expert_cv_min": 0.5563065094987957, "raw_expert_cv_n": 120, "raw_expert_imax_ceiling": 8.0, "raw_expert_imax_hit_ceiling": 1, "raw_expert_imax_max": 8.0, - "raw_expert_imax_mean": 6.812005615234375, - "raw_expert_imax_min": 2.1181640625, + "raw_expert_imax_mean": 6.808540852864583, + "raw_expert_imax_min": 2.200439453125, "raw_expert_imax_n": 120, "records": { "drop_dispatch": 960, @@ -251,9 +251,9 @@ "seq_length": "4096", "shared_expert_overlap": "0", "topk": "8", - "valid_max_rank_reduction_max": 0.7486673262922141, - "valid_max_rank_reduction_mean": 0.497309315009654, - "valid_max_rank_reduction_min": 0.11442624722987947, + "valid_max_rank_reduction_max": 0.7486191236037806, + "valid_max_rank_reduction_mean": 0.4949595505718118, + "valid_max_rank_reduction_min": 0.004617253948967193, "valid_max_rank_reduction_n": 120, "zero_drop": 1 }, @@ -285,9 +285,9 @@ "force_balance": "1", "global_batch_size": "8", "global_snapshots": 120, - "hottest_expert_token_fraction_max": 0.1317138671875, - "hottest_expert_token_fraction_mean": 0.129486083984375, - "hottest_expert_token_fraction_min": 0.128021240234375, + "hottest_expert_token_fraction_max": 0.13189697265625, + "hottest_expert_token_fraction_mean": 0.1293980916341146, + "hottest_expert_token_fraction_min": 0.1279296875, "hottest_expert_token_fraction_n": 120, "incomplete_snapshots": 0, "malformed_records": 0, @@ -303,23 +303,23 @@ "post_physical_rank_imax_mean": 1.0, "post_physical_rank_imax_min": 1.0, "post_physical_rank_imax_n": 120, - "pre_home_rank_cv_max": 0.01200287811464368, - "pre_home_rank_cv_mean": 0.0057851448972811755, - "pre_home_rank_cv_min": 0.0010965114882337452, + "pre_home_rank_cv_max": 0.01039906345549811, + "pre_home_rank_cv_mean": 0.004760489966476766, + "pre_home_rank_cv_min": 0.002294812301292466, "pre_home_rank_cv_n": 120, - "pre_home_rank_imax_max": 1.015289306640625, - "pre_home_rank_imax_mean": 1.0091585795084634, - "pre_home_rank_imax_min": 1.001373291015625, + "pre_home_rank_imax_max": 1.015838623046875, + "pre_home_rank_imax_mean": 1.0073875427246093, + "pre_home_rank_imax_min": 1.002899169921875, "pre_home_rank_imax_n": 120, - "raw_expert_cv_max": 0.01846260511411008, - "raw_expert_cv_mean": 0.01497261060068789, - "raw_expert_cv_min": 0.011926166451983852, + "raw_expert_cv_max": 0.018080370892918116, + "raw_expert_cv_mean": 0.014602030835140035, + "raw_expert_cv_min": 0.011942398269672016, "raw_expert_cv_n": 120, "raw_expert_imax_ceiling": 8.0, "raw_expert_imax_hit_ceiling": 0, - "raw_expert_imax_max": 1.0537109375, - "raw_expert_imax_mean": 1.035888671875, - "raw_expert_imax_min": 1.024169921875, + "raw_expert_imax_max": 1.05517578125, + "raw_expert_imax_mean": 1.0351847330729167, + "raw_expert_imax_min": 1.0234375, "raw_expert_imax_n": 120, "records": { "drop_dispatch": 960, @@ -337,9 +337,9 @@ "seq_length": "4096", "shared_expert_overlap": "0", "topk": "8", - "valid_max_rank_reduction_max": 0.01505906399350747, - "valid_max_rank_reduction_mean": 0.009067283410582577, - "valid_max_rank_reduction_min": 0.001371407673787828, + "valid_max_rank_reduction_max": 0.015591672424670291, + "valid_max_rank_reduction_mean": 0.007327065584253315, + "valid_max_rank_reduction_min": 0.0028907890332592885, "valid_max_rank_reduction_n": 120, "zero_drop": 1 }, @@ -351,8 +351,8 @@ "branch": "token-drop-analysis", "capacity": "2.00", "capacity_mode": "static", - "cold_expert_fraction_max": 0.7890625, - "cold_expert_fraction_mean": 0.4895833333333333, + "cold_expert_fraction_max": 0.7734375, + "cold_expert_fraction_mean": 0.4877387152777778, "cold_expert_fraction_min": 0.0, "cold_expert_fraction_n": 144, "commit": "b099fd84", @@ -372,7 +372,7 @@ "global_batch_size": "8", "global_snapshots": 144, "hottest_expert_token_fraction_max": 0.9999542236328125, - "hottest_expert_token_fraction_mean": 0.9661488003200955, + "hottest_expert_token_fraction_mean": 0.9661530388726128, "hottest_expert_token_fraction_min": 0.546875, "hottest_expert_token_fraction_n": 144, "incomplete_snapshots": 0, @@ -389,22 +389,22 @@ "post_physical_rank_imax_mean": 1.0, "post_physical_rank_imax_min": 1.0, "post_physical_rank_imax_n": 144, - "pre_home_rank_cv_max": 1.5534351281913334, - "pre_home_rank_cv_mean": 0.8358247018998112, + "pre_home_rank_cv_max": 1.5481766462033006, + "pre_home_rank_cv_mean": 0.833177141336639, "pre_home_rank_cv_min": 0.3167748701963148, "pre_home_rank_cv_n": 144, - "pre_home_rank_imax_max": 4.9171295166015625, - "pre_home_rank_imax_mean": 2.5417197545369468, + "pre_home_rank_imax_max": 4.8278045654296875, + "pre_home_rank_imax_mean": 2.5318915049235025, "pre_home_rank_imax_min": 1.2317352294921875, "pre_home_rank_imax_n": 144, - "raw_expert_cv_max": 3.8719906493286524, - "raw_expert_cv_mean": 3.456592832253774, + "raw_expert_cv_max": 3.8719827682969976, + "raw_expert_cv_mean": 3.445471383880661, "raw_expert_cv_min": 1.5070684637391225, "raw_expert_cv_n": 144, "raw_expert_imax_ceiling": 16.0, "raw_expert_imax_hit_ceiling": 1, "raw_expert_imax_max": 15.999267578125, - "raw_expert_imax_mean": 15.458380805121529, + "raw_expert_imax_mean": 15.458448621961805, "raw_expert_imax_min": 8.75, "raw_expert_imax_n": 144, "records": { @@ -423,8 +423,8 @@ "seq_length": "8192", "shared_expert_overlap": "0", "topk": "8", - "valid_max_rank_reduction_max": 0.7966293145983385, - "valid_max_rank_reduction_mean": 0.5769520293233497, + "valid_max_rank_reduction_max": 0.7928665117969627, + "valid_max_rank_reduction_mean": 0.5755050085258977, "valid_max_rank_reduction_min": 0.18813720996494185, "valid_max_rank_reduction_n": 144, "zero_drop": 1 @@ -523,8 +523,8 @@ "branch": "token-drop-analysis", "capacity": "2.00", "capacity_mode": "static", - "cold_expert_fraction_max": 0.9296875, - "cold_expert_fraction_mean": 0.6222466362847222, + "cold_expert_fraction_max": 0.875, + "cold_expert_fraction_mean": 0.6207817925347222, "cold_expert_fraction_min": 0.0, "cold_expert_fraction_n": 576, "commit": "b099fd84", @@ -543,8 +543,8 @@ "force_balance": "0", "global_batch_size": "8", "global_snapshots": 576, - "hottest_expert_token_fraction_max": 1.0, - "hottest_expert_token_fraction_mean": 0.9104721811082628, + "hottest_expert_token_fraction_max": 0.999969482421875, + "hottest_expert_token_fraction_mean": 0.9099888271755643, "hottest_expert_token_fraction_min": 0.17547607421875, "hottest_expert_token_fraction_n": 576, "incomplete_snapshots": 0, @@ -561,22 +561,22 @@ "post_physical_rank_imax_mean": 1.0, "post_physical_rank_imax_min": 1.0, "post_physical_rank_imax_n": 576, - "pre_home_rank_cv_max": 1.5704882864247645, - "pre_home_rank_cv_mean": 0.7589291473184087, + "pre_home_rank_cv_max": 1.5779242846760857, + "pre_home_rank_cv_mean": 0.7516915898666823, "pre_home_rank_cv_min": 0.13794675824562003, "pre_home_rank_cv_n": 576, - "pre_home_rank_imax_max": 4.974578857421875, - "pre_home_rank_imax_mean": 2.3463818762037487, + "pre_home_rank_imax_max": 4.995025634765625, + "pre_home_rank_imax_mean": 2.3228725327385797, "pre_home_rank_imax_min": 1.2030029296875, "pre_home_rank_imax_n": 576, - "raw_expert_cv_max": 3.8726526007751483, - "raw_expert_cv_mean": 3.3062029242663797, + "raw_expert_cv_max": 3.8728572743263237, + "raw_expert_cv_mean": 3.2880310327050792, "raw_expert_cv_min": 0.540439815722741, "raw_expert_cv_n": 576, "raw_expert_imax_ceiling": 16.0, "raw_expert_imax_hit_ceiling": 1, - "raw_expert_imax_max": 16.0, - "raw_expert_imax_mean": 14.567554897732204, + "raw_expert_imax_max": 15.99951171875, + "raw_expert_imax_mean": 14.559821234809029, "raw_expert_imax_min": 2.8076171875, "raw_expert_imax_n": 576, "records": { @@ -595,8 +595,8 @@ "seq_length": "4096", "shared_expert_overlap": "0", "topk": "8", - "valid_max_rank_reduction_max": 0.7989779580018036, - "valid_max_rank_reduction_mean": 0.5410548593820177, + "valid_max_rank_reduction_max": 0.7998008272390135, + "valid_max_rank_reduction_mean": 0.5374768231729111, "valid_max_rank_reduction_min": 0.16874682902080163, "valid_max_rank_reduction_n": 576, "zero_drop": 1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv index 9365b210c63c49305a90cdd021aa0b9fa4b133fb..a3c6e81078b00862e686e0dfc98ff939e9afac2e 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -1,9 +1,9 @@ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_expected_assignments drop_rate_pct_overall echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records observed_dropped_tokens post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 natural DSV2_LITE_L24 echo 64 6 8 24 23 1 8192 8 static 2.00 0 0 token-drop-analysis b099fd84 seq_aux_loss 1e-3 0 0 0.59375 0.12777400362318841 0.0 276 108527616.0 0.0 0 0 0.5 0.5 0.5 276 276 0.999969482421875 0.6977317920629529 0.18292236328125 276 0 0 0.0 0.0 0.0 0.0 276 1.0 1.0 1.0 276 1.3955462296743573 0.5956958266809962 0.06838324069662365 276 4.433919270833333 2.117264752226751 1.1109822591145833 276 3.0650012939743223 1.7377059172733105 0.36860618049647703 276 10.666666666666666 1 10.666341145833334 7.442472448671498 1.951171875 276 1.0 1.0 1.0 2208 0.7744658982453564 0.4729061659118927 0.09989561777794055 276 1 DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 forced DSV2_LITE_L24 echo 64 6 8 24 23 1 8192 8 static 2.00 1 0 token-drop-analysis b099fd84 seq_aux_loss 1e-3 0 0 0.0 0.0 0.0 276 108527616.0 0.0 0 0 0.5 0.5 0.5 276 276 0.0982818603515625 0.09644090956535892 0.09539794921875 276 0 0 0.0 0.0 0.0 0.0 276 1.0 1.0 1.0 276 0.006715970721309454 0.00392157619549587 0.0014708009315538319 276 1.0122884114583333 1.0060781580238527 1.00213623046875 276 0.015123587213954862 0.012112028161640214 0.008171875787738593 276 10.666666666666666 0 1.04833984375 1.0287030353638287 1.017578125 276 1.0 1.0 1.0 2208 0.01213923948870488 0.006037272492700937 0.0021316767159997563 276 1 -QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 natural QWEN2_57B echo 64 8 8 10 10 0 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.828125 0.3279947916666667 0.0 120 31457280.0 0.0 0 0 0.5 0.5 0.5 120 120 1.0 0.8515007019042968 0.2647705078125 120 0 0 0.0 0.0 0.0 0.0 120 1.0 1.0 1.0 120 1.2169604837256403 0.6377553523750428 0.09633592940981593 120 3.978790283203125 2.1783960978190104 1.12921142578125 120 2.6426244377115378 1.9430158591244282 0.5540860535283595 120 8.0 1 8.0 6.812005615234375 2.1181640625 120 1.0 1.0 1.0 960 0.7486673262922141 0.497309315009654 0.11442624722987947 120 1 -QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 forced QWEN2_57B echo 64 8 8 10 10 0 4096 8 static 2.00 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 120 31457280.0 0.0 0 0 0.5 0.5 0.5 120 120 0.1317138671875 0.129486083984375 0.128021240234375 120 0 0 0.0 0.0 0.0 0.0 120 1.0 1.0 1.0 120 0.01200287811464368 0.0057851448972811755 0.0010965114882337452 120 1.015289306640625 1.0091585795084634 1.001373291015625 120 0.01846260511411008 0.01497261060068789 0.011926166451983852 120 8.0 0 1.0537109375 1.035888671875 1.024169921875 120 1.0 1.0 1.0 960 0.01505906399350747 0.009067283410582577 0.001371407673787828 120 1 -QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 natural QWEN3_235B echo 128 8 16 12 12 0 8192 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.7890625 0.4895833333333333 0.0 144 75497472.0 0.0 0 0 0.5 0.5 0.5 144 144 0.9999542236328125 0.9661488003200955 0.546875 144 0 0 0.0 0.0 0.0 0.0 144 1.0 1.0 1.0 144 1.5534351281913334 0.8358247018998112 0.3167748701963148 144 4.9171295166015625 2.5417197545369468 1.2317352294921875 144 3.8719906493286524 3.456592832253774 1.5070684637391225 144 16.0 1 15.999267578125 15.458380805121529 8.75 144 1.0 1.0 1.0 1152 0.7966293145983385 0.5769520293233497 0.18813720996494185 144 1 +QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 natural QWEN2_57B echo 64 8 8 10 10 0 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.796875 0.33098958333333334 0.0 120 31457280.0 0.0 0 0 0.5 0.5 0.5 120 120 1.0 0.8510676066080729 0.275054931640625 120 0 0 0.0 0.0 0.0 0.0 120 1.0 1.0 1.0 120 1.2755844916699237 0.6343602950592665 0.0020009968962783516 120 3.97802734375 2.162261708577474 1.004638671875 120 2.643741260408719 1.9491049129110607 0.5563065094987957 120 8.0 1 8.0 6.808540852864583 2.200439453125 120 1.0 1.0 1.0 960 0.7486191236037806 0.4949595505718118 0.004617253948967193 120 1 +QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 forced QWEN2_57B echo 64 8 8 10 10 0 4096 8 static 2.00 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 120 31457280.0 0.0 0 0 0.5 0.5 0.5 120 120 0.13189697265625 0.1293980916341146 0.1279296875 120 0 0 0.0 0.0 0.0 0.0 120 1.0 1.0 1.0 120 0.01039906345549811 0.004760489966476766 0.002294812301292466 120 1.015838623046875 1.0073875427246093 1.002899169921875 120 0.018080370892918116 0.014602030835140035 0.011942398269672016 120 8.0 0 1.05517578125 1.0351847330729167 1.0234375 120 1.0 1.0 1.0 960 0.015591672424670291 0.007327065584253315 0.0028907890332592885 120 1 +QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 natural QWEN3_235B echo 128 8 16 12 12 0 8192 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.7734375 0.4877387152777778 0.0 144 75497472.0 0.0 0 0 0.5 0.5 0.5 144 144 0.9999542236328125 0.9661530388726128 0.546875 144 0 0 0.0 0.0 0.0 0.0 144 1.0 1.0 1.0 144 1.5481766462033006 0.833177141336639 0.3167748701963148 144 4.8278045654296875 2.5318915049235025 1.2317352294921875 144 3.8719827682969976 3.445471383880661 1.5070684637391225 144 16.0 1 15.999267578125 15.458448621961805 8.75 144 1.0 1.0 1.0 1152 0.7928665117969627 0.5755050085258977 0.18813720996494185 144 1 QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 forced QWEN3_235B echo 128 8 16 12 12 0 8192 8 static 2.00 1 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.0 0.0 0.0 144 75497472.0 0.0 0 0 0.5 0.5 0.5 144 144 0.0663299560546875 0.0649354722764757 0.0641632080078125 144 0 0 0.0 0.0 0.0 0.0 144 1.0 1.0 1.0 144 0.006990045273933762 0.0034121784262079076 0.0012782366924169522 144 1.0104522705078125 1.0053127076890733 1.0019378662109375 144 0.017577383237659006 0.015149663454502107 0.013421143735031664 144 16.0 0 1.061279296875 1.0389675564236112 1.026611328125 144 1.0 1.0 1.0 1152 0.010344150647075702 0.005281131647695585 0.0019341181487291168 144 1 -QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 natural QWEN3_30B echo 128 8 16 48 48 0 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.9296875 0.6222466362847222 0.0 576 150994944.0 0.0 0 0 0.5 0.5 0.5 576 576 1.0 0.9104721811082628 0.17547607421875 576 0 0 0.0 0.0 0.0 0.0 576 1.0 1.0 1.0 576 1.5704882864247645 0.7589291473184087 0.13794675824562003 576 4.974578857421875 2.3463818762037487 1.2030029296875 576 3.8726526007751483 3.3062029242663797 0.540439815722741 576 16.0 1 16.0 14.567554897732204 2.8076171875 576 1.0 1.0 1.0 4608 0.7989779580018036 0.5410548593820177 0.16874682902080163 576 1 +QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 natural QWEN3_30B echo 128 8 16 48 48 0 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.875 0.6207817925347222 0.0 576 150994944.0 0.0 0 0 0.5 0.5 0.5 576 576 0.999969482421875 0.9099888271755643 0.17547607421875 576 0 0 0.0 0.0 0.0 0.0 576 1.0 1.0 1.0 576 1.5779242846760857 0.7516915898666823 0.13794675824562003 576 4.995025634765625 2.3228725327385797 1.2030029296875 576 3.8728572743263237 3.2880310327050792 0.540439815722741 576 16.0 1 15.99951171875 14.559821234809029 2.8076171875 576 1.0 1.0 1.0 4608 0.7998008272390135 0.5374768231729111 0.16874682902080163 576 1 QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 forced QWEN3_30B echo 128 8 16 48 48 0 4096 8 static 2.00 1 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.0 0.0 0.0 576 150994944.0 0.0 0 0 0.5 0.5 0.5 576 576 0.06854248046875 0.066029601626926 0.06488037109375 576 0 0 0.0 0.0 0.0 0.0 576 1.0 1.0 1.0 576 0.009112333689945906 0.004932308997139817 0.0012089160228135533 576 1.01739501953125 1.0077196756998699 1.00177001953125 576 0.02488528396573193 0.021371222454350164 0.018128366157757106 576 16.0 0 1.0966796875 1.056473626030816 1.0380859375 576 1.0 1.0 1.0 4608 0.017097606335113084 0.00765283599899436 0.0017668920977274112 576 1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv index 8d3269804c48f77299a831e40f9a8a6890dc6323..4690daf3aa9c95ba5457034396d33581dd4912b0 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv @@ -6,3 +6,4 @@ QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 963.67 920.30 133533 /w DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 806.56 800.50 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 DSV2_LITE_L24 echo matched 8 8192 8 2.00 none+forcebal 1 OK 799.97 793.80 140541 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 QWEN2_57B echo matched 8 4096 8 2.00 none 1 OK 286.46 282.70 109133 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN2_57B echo matched 8 4096 8 2.00 none+forcebal 1 OK 276.56 271.10 109131 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1